From 88646e13ed02bd7e886da5eee0437b5b3290012d Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 7 Dec 2018 09:34:13 +0000 Subject: [PATCH 01/41] chore: update dependencies License: MIT Signed-off-by: Alan Shaw --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index d1f51beaab..43f381120e 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "libp2p": "~0.24.1", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.14.1", - "libp2p-kad-dht": "~0.12.1", + "libp2p-kad-dht": "~0.13.0", "libp2p-keychain": "~0.3.3", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.4", @@ -145,7 +145,7 @@ "mafmt": "^6.0.2", "mime-types": "^2.1.21", "mkdirp": "~0.5.1", - "multiaddr": "^6.0.0", + "multiaddr": "~6.0.0", "multiaddr-to-uri": "^4.0.0", "multibase": "~0.6.0", "multihashes": "~0.4.14", From b57865df16c14c255b8d68309b363a747471dcda Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 12 Dec 2018 08:59:44 +0000 Subject: [PATCH 02/41] chore: use hat License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43f381120e..220b60fa0f 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,7 @@ "mafmt": "^6.0.2", "mime-types": "^2.1.21", "mkdirp": "~0.5.1", - "multiaddr": "~6.0.0", + "multiaddr": "^6.0.0", "multiaddr-to-uri": "^4.0.0", "multibase": "~0.6.0", "multihashes": "~0.4.14", From d7d9f164ba61f538198711921553cc46465aa08b Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 12 Dec 2018 09:00:48 +0000 Subject: [PATCH 03/41] fix: revert kad dep upgrade License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 220b60fa0f..d1f51beaab 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "libp2p": "~0.24.1", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.14.1", - "libp2p-kad-dht": "~0.13.0", + "libp2p-kad-dht": "~0.12.1", "libp2p-keychain": "~0.3.3", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.4", From 0d9cfe4dd6d74a382084428a1614d5fdcc6678db Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 7 Dec 2018 09:34:13 +0000 Subject: [PATCH 04/41] chore: update dependencies License: MIT Signed-off-by: Alan Shaw --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index d1f51beaab..43f381120e 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "libp2p": "~0.24.1", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.14.1", - "libp2p-kad-dht": "~0.12.1", + "libp2p-kad-dht": "~0.13.0", "libp2p-keychain": "~0.3.3", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.4", @@ -145,7 +145,7 @@ "mafmt": "^6.0.2", "mime-types": "^2.1.21", "mkdirp": "~0.5.1", - "multiaddr": "^6.0.0", + "multiaddr": "~6.0.0", "multiaddr-to-uri": "^4.0.0", "multibase": "~0.6.0", "multihashes": "~0.4.14", From 70613b3735ece58f63c193432d58940233fe9543 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 12 Dec 2018 08:59:44 +0000 Subject: [PATCH 05/41] chore: use hat License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 43f381120e..220b60fa0f 100644 --- a/package.json +++ b/package.json @@ -145,7 +145,7 @@ "mafmt": "^6.0.2", "mime-types": "^2.1.21", "mkdirp": "~0.5.1", - "multiaddr": "~6.0.0", + "multiaddr": "^6.0.0", "multiaddr-to-uri": "^4.0.0", "multibase": "~0.6.0", "multihashes": "~0.4.14", From dde12bc7c66077849f3a46b94b886909b2412656 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 12 Dec 2018 09:00:48 +0000 Subject: [PATCH 06/41] fix: revert kad dep upgrade License: MIT Signed-off-by: Alan Shaw --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 220b60fa0f..d1f51beaab 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "libp2p": "~0.24.1", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.14.1", - "libp2p-kad-dht": "~0.13.0", + "libp2p-kad-dht": "~0.12.1", "libp2p-keychain": "~0.3.3", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.4", From 0011cc1c6de1b905a312a1431329b0961dff9ca1 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Wed, 29 Aug 2018 17:25:35 +0100 Subject: [PATCH 07/41] feat: add cid-base option License: MIT Signed-off-by: Alan Shaw --- src/cli/commands/add.js | 15 +- src/cli/commands/bitswap/stat.js | 16 +- src/cli/commands/bitswap/unwant.js | 11 +- src/cli/commands/bitswap/wantlist.js | 16 +- src/cli/commands/block/put.js | 11 +- src/cli/commands/block/stat.js | 16 +- src/cli/commands/ls.js | 29 ++-- src/cli/commands/object/get.js | 13 +- src/cli/commands/object/links.js | 11 +- src/cli/commands/object/new.js | 14 +- src/cli/commands/object/patch/add-link.js | 11 +- src/cli/commands/object/patch/append-data.js | 23 ++- src/cli/commands/object/patch/rm-link.js | 14 +- src/cli/commands/object/patch/set-data.js | 23 ++- src/cli/commands/object/put.js | 19 ++- src/cli/commands/pin/add.js | 13 +- src/cli/commands/pin/ls.js | 13 +- src/cli/commands/pin/rm.js | 16 +- src/cli/commands/resolve.js | 9 +- src/core/components/bitswap.js | 67 +++++--- src/core/components/block.js | 11 +- .../files-regular/add-pull-stream.js | 20 ++- .../files-regular/ls-pull-stream.js | 3 +- src/core/components/object.js | 5 +- src/core/components/pin.js | 43 ++++- src/core/components/resolve.js | 5 +- src/http/api/resources/bitswap.js | 35 +++- src/http/api/resources/block.js | 21 ++- src/http/api/resources/files-regular.js | 17 +- src/http/api/resources/object.js | 156 +++++++++++++----- src/http/api/resources/pin.js | 39 ++++- src/http/api/resources/resolve.js | 9 +- src/utils/cid.js | 14 ++ test/cli/bitswap.js | 15 ++ test/cli/block.js | 24 +++ test/cli/files.js | 15 ++ test/cli/ls.js | 15 ++ test/cli/object.js | 77 +++++++++ test/cli/pin.js | 25 +++ test/cli/resolve.js | 24 ++- 40 files changed, 726 insertions(+), 207 deletions(-) create mode 100644 src/utils/cid.js diff --git a/src/cli/commands/add.js b/src/cli/commands/add.js index 8b930434d6..9ea059b3c2 100644 --- a/src/cli/commands/add.js +++ b/src/cli/commands/add.js @@ -11,9 +11,8 @@ const getFolderSize = require('get-folder-size') const byteman = require('byteman') const waterfall = require('async/waterfall') const mh = require('multihashes') -const utils = require('../utils') -const print = require('../utils').print -const createProgressBar = require('../utils').createProgressBar +const multibase = require('multibase') +const { print, isDaemonOn, createProgressBar } = require('../../utils') function checkPath (inPath, recursive) { // This function is to check for the following possible inputs @@ -156,10 +155,15 @@ module.exports = { describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)', default: 0 }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names + }, hash: { type: 'string', choices: Object.keys(mh.names), - describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)' + describe: 'Hash function to use. Will set CID version to 1 if used. (experimental)' }, quiet: { alias: 'q', @@ -194,6 +198,7 @@ module.exports = { ? argv.shardSplitThreshold : Infinity, cidVersion: argv.cidVersion, + cidBase: argv.cidBase, rawLeaves: argv.rawLeaves, onlyHash: argv.onlyHash, hashAlg: argv.hash, @@ -202,7 +207,7 @@ module.exports = { chunker: argv.chunker } - if (options.enableShardingExperiment && utils.isDaemonOn()) { + if (options.enableShardingExperiment && isDaemonOn()) { throw new Error('Error: Enabling the sharding experiment should be done on the daemon') } const ipfs = argv.ipfs diff --git a/src/cli/commands/bitswap/stat.js b/src/cli/commands/bitswap/stat.js index d7a26680f1..7dd01c5946 100644 --- a/src/cli/commands/bitswap/stat.js +++ b/src/cli/commands/bitswap/stat.js @@ -1,22 +1,30 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') module.exports = { command: 'stat', describe: 'Show some diagnostic information on the bitswap agent.', - builder: {}, + builder: { + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names + } + }, handler (argv) { - argv.ipfs.bitswap.stat((err, stats) => { + const { cidBase } = argv + + argv.ipfs.bitswap.stat({ cidBase }, (err, stats) => { if (err) { throw err } stats.wantlist = stats.wantlist || [] - stats.wantlist = stats.wantlist.map(entry => entry['/']) stats.peers = stats.peers || [] print(`bitswap status diff --git a/src/cli/commands/bitswap/unwant.js b/src/cli/commands/bitswap/unwant.js index ea676450fa..810f7946d4 100644 --- a/src/cli/commands/bitswap/unwant.js +++ b/src/cli/commands/bitswap/unwant.js @@ -1,6 +1,8 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'unwant ', @@ -12,6 +14,11 @@ module.exports = { alias: 'k', describe: 'Key to remove from your wantlist', type: 'string' + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, handler (argv) { @@ -19,7 +26,7 @@ module.exports = { if (err) { throw err } - print(`Key ${argv.key} removed from wantlist`) + print(`Key ${cidToString(argv.key, argv.cidBase)} removed from wantlist`) }) } } diff --git a/src/cli/commands/bitswap/wantlist.js b/src/cli/commands/bitswap/wantlist.js index 9397f492ae..1f9bda315f 100644 --- a/src/cli/commands/bitswap/wantlist.js +++ b/src/cli/commands/bitswap/wantlist.js @@ -1,6 +1,7 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') module.exports = { command: 'wantlist [peer]', @@ -12,17 +13,22 @@ module.exports = { alias: 'p', describe: 'Specify which peer to show wantlist for.', type: 'string' + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, handler (argv) { - argv.ipfs.bitswap.wantlist(argv.peer, (err, res) => { + const { peer, cidBase } = argv + + argv.ipfs.bitswap.wantlist(peer, { cidBase }, (err, cids) => { if (err) { throw err } - res.Keys.forEach((cid) => { - print(cid['/']) - }) + cids.forEach((cid) => print(cid)) }) } } diff --git a/src/cli/commands/block/put.js b/src/cli/commands/block/put.js index 9d75a9c72f..1606ee6ded 100644 --- a/src/cli/commands/block/put.js +++ b/src/cli/commands/block/put.js @@ -6,7 +6,9 @@ const bl = require('bl') const fs = require('fs') const Block = require('ipfs-block') const waterfall = require('async/waterfall') -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') function addBlock (data, opts) { const ipfs = opts.ipfs @@ -27,7 +29,7 @@ function addBlock (data, opts) { if (err) { throw err } - print(cid.toBaseEncodedString()) + print(cidToString(cid, opts.cidBase)) }) } @@ -54,6 +56,11 @@ module.exports = { describe: 'cid version', type: 'number', default: 0 + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, diff --git a/src/cli/commands/block/stat.js b/src/cli/commands/block/stat.js index 07190fd225..330f4e544d 100644 --- a/src/cli/commands/block/stat.js +++ b/src/cli/commands/block/stat.js @@ -1,17 +1,25 @@ 'use strict' -const CID = require('cids') -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') module.exports = { command: 'stat ', describe: 'Print information of a raw IPFS block', - builder: {}, + builder: { + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names + } + }, handler (argv) { - argv.ipfs.block.stat(new CID(argv.key), (err, stats) => { + const { key, cidBase } = argv + + argv.ipfs.block.stat(key, { cidBase }, (err, stats) => { if (err) { throw err } diff --git a/src/cli/commands/ls.js b/src/cli/commands/ls.js index 97ba7ca692..cb882836fb 100644 --- a/src/cli/commands/ls.js +++ b/src/cli/commands/ls.js @@ -1,6 +1,7 @@ 'use strict' -const utils = require('../utils') +const multibase = require('multibase') +const { print, rightpad } = require('../utils') module.exports = { command: 'ls ', @@ -24,16 +25,16 @@ module.exports = { desc: 'Resolve linked objects to find out their types. (not implemented yet)', type: 'boolean', default: false // should be true when implemented + }, + 'cid-base': { + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, handler (argv) { - let path = argv.key - if (path.startsWith('/ipfs/')) { - path = path.replace('/ipfs/', '') - } - - argv.ipfs.ls(path, { recursive: argv.recursive }, (err, links) => { + argv.ipfs.ls(argv.key, { recursive: argv.recursive, cidBase: argv.cidBase }, (err, links) => { if (err) { throw err } @@ -45,12 +46,18 @@ module.exports = { const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length)) const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length)) + let pathParts = argv.key.split('/') + + if (argv.key.startsWith('/ipfs/')) { + pathParts = pathParts.slice(2) + } + links.forEach(link => { const fileName = link.type === 'dir' ? `${link.name || ''}/` : link.name - const padding = link.depth - path.split('/').length - utils.print( - utils.rightpad(link.hash, multihashWidth + 1) + - utils.rightpad(link.size || '', sizeWidth + 1) + + const padding = link.depth - pathParts.length + print( + rightpad(link.hash, multihashWidth + 1) + + rightpad(link.size || '', sizeWidth + 1) + ' '.repeat(padding) + fileName ) }) diff --git a/src/cli/commands/object/get.js b/src/cli/commands/object/get.js index 548ae4f508..45d8604633 100644 --- a/src/cli/commands/object/get.js +++ b/src/cli/commands/object/get.js @@ -1,7 +1,9 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') const dagPB = require('ipld-dag-pb') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'get ', @@ -14,8 +16,9 @@ module.exports = { default: 'base64' }, 'cid-base': { - default: 'base58btc', - describe: 'CID base to use.' + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, @@ -38,13 +41,13 @@ module.exports = { const answer = { Data: data, - Hash: result.toBaseEncodedString(argv.cidBase), + Hash: cidToString(result, argv.cidBase), Size: node.size, Links: node.links.map((l) => { return { Name: l.name, Size: l.size, - Hash: l.cid.toBaseEncodedString(argv.cidBase) + Hash: cidToString(l.cid, argv.cidBase) } }) } diff --git a/src/cli/commands/object/links.js b/src/cli/commands/object/links.js index 9a3a3e203e..0fd98906ef 100644 --- a/src/cli/commands/object/links.js +++ b/src/cli/commands/object/links.js @@ -1,6 +1,8 @@ 'use strict' -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'links ', @@ -9,8 +11,9 @@ module.exports = { builder: { 'cid-base': { - default: 'base58btc', - describe: 'CID base to use.' + describe: 'Number base to display CIDs in.', + type: 'string', + choices: multibase.names } }, @@ -23,7 +26,7 @@ module.exports = { } links.forEach((link) => { - print(`${link.cid.toBaseEncodedString(argv.cidBase)} ${link.size} ${link.name}`) + print(`${cidToString(link.cid, argv.cidBase)} ${link.size} ${link.name}`) }) }) } diff --git a/src/cli/commands/object/new.js b/src/cli/commands/object/new.js index fc8543fd25..1304997691 100644 --- a/src/cli/commands/object/new.js +++ b/src/cli/commands/object/new.js @@ -1,9 +1,8 @@ 'use strict' -const debug = require('debug') -const log = debug('cli:object') -log.error = debug('cli:object:error') -const print = require('../../utils').print +const multibase = require('multibase') +const { print } = require('../../utils') +const { cidToString } = require('../../../utils/cid') module.exports = { command: 'new [