diff --git a/docs/core-api/DHT.md b/docs/core-api/DHT.md index fcb8d3a023..a35181541e 100644 --- a/docs/core-api/DHT.md +++ b/docs/core-api/DHT.md @@ -126,7 +126,7 @@ A great source of [examples][] can be found in the tests for this API. | Name | Type | Description | | ---- | ---- | ----------- | -| key | Buffer | The key associated with the value to find | +| key | `Buffer` or `string` | The key associated with the value to find | ### Options diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index 50253ec44b..5b0e61cb20 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -916,7 +916,6 @@ An optional object which may have the following keys: | Name | Type | Default | Description | | ---- | ---- | ------- | ----------- | -| sort | `boolean` | `false` | If true entries will be sorted by filename | | timeout | `Number` | `undefined` | A timeout in ms | | signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | diff --git a/docs/core-api/KEY.md b/docs/core-api/KEY.md index 11a6b6685d..6d2d5da60b 100644 --- a/docs/core-api/KEY.md +++ b/docs/core-api/KEY.md @@ -289,4 +289,4 @@ A great source of [examples][] can be found in the tests for this API. [examples]: https://github.com/ipfs/js-ipfs/blob/master/packages/interface-ipfs-core/src/key [cid]: https://www.npmjs.com/package/cids -[AbortSignal]: https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal \ No newline at end of file +[AbortSignal]: https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal diff --git a/packages/interface-ipfs-core/src/bootstrap/rm.js b/packages/interface-ipfs-core/src/bootstrap/rm.js index 81b7bf41e8..d4f43e0318 100644 --- a/packages/interface-ipfs-core/src/bootstrap/rm.js +++ b/packages/interface-ipfs-core/src/bootstrap/rm.js @@ -61,7 +61,7 @@ module.exports = (common, options) => { const rmRes = await ipfs.bootstrap.rm(null, { all: true }) const removedPeers = rmRes.Peers - expect(removedPeers).to.eql(addedPeers) + expect(removedPeers.sort()).to.deep.equal(addedPeers.sort()) }) }) } diff --git a/packages/interface-ipfs-core/src/dht/get.js b/packages/interface-ipfs-core/src/dht/get.js index a886c3be8a..21efdf240d 100644 --- a/packages/interface-ipfs-core/src/dht/get.js +++ b/packages/interface-ipfs-core/src/dht/get.js @@ -1,10 +1,9 @@ /* eslint-env mocha */ 'use strict' -const { Buffer } = require('buffer') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') -const drain = require('it-drain') +const all = require('it-all') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -15,9 +14,7 @@ module.exports = (common, options) => { const describe = getDescribe(options) const it = getIt(options) - describe.skip('.dht.get', function () { - this.timeout(80 * 1000) - + describe('.dht.get', function () { let nodeA let nodeB @@ -30,12 +27,10 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when getting a value from the DHT', async () => { - const key = Buffer.from('/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - const data = Buffer.from('data') - - await drain(nodeA.dht.put(key, data, { verbose: true })) + const [data] = await all(nodeA.add('should put a value to the DHT')) + const publish = await nodeA.name.publish(data.cid) - await testTimeout(() => nodeB.dht.get(key, { + await testTimeout(() => nodeB.dht.get(`/ipns/${publish.name}`, { timeout: 1 })) }) @@ -46,13 +41,11 @@ module.exports = (common, options) => { }) it('should get a value after it was put on another node', async () => { - const key = Buffer.from('/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - const value = Buffer.from('data') - - await drain(nodeB.dht.put(key, value)) - const result = await nodeA.dht.get(key) + const [data] = await all(nodeA.add('should put a value to the DHT')) + const publish = await nodeA.name.publish(data.cid) + const record = await nodeA.dht.get(`/ipns/${publish.name}`) - expect(result).to.eql(value) + expect(record.toString()).to.contain(data.cid.toString()) }) }) } diff --git a/packages/interface-ipfs-core/src/dht/put.js b/packages/interface-ipfs-core/src/dht/put.js index 3c579e9d20..3b63376780 100644 --- a/packages/interface-ipfs-core/src/dht/put.js +++ b/packages/interface-ipfs-core/src/dht/put.js @@ -2,10 +2,10 @@ 'use strict' const { Buffer } = require('buffer') -const { getDescribe, getIt } = require('../utils/mocha') -const drain = require('it-drain') +const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') const CID = require('cids') +const all = require('it-all') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -16,10 +16,7 @@ module.exports = (common, options) => { const describe = getDescribe(options) const it = getIt(options) - // TODO unskip this after go-ipfs 0.5.0 ships interface is going to change - describe.skip('.dht.put', function () { - this.timeout(80 * 1000) - + describe('.dht.put', function () { let nodeA let nodeB @@ -38,12 +35,13 @@ module.exports = (common, options) => { }) it('should put a value to the DHT', async function () { - this.timeout(80 * 1000) - - const key = Buffer.from('/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - const data = Buffer.from('data') - - await drain(nodeA.dht.put(key, data, { verbose: true })) + const [data] = await all(nodeA.add('should put a value to the DHT')) + const publish = await nodeA.name.publish(data.cid) + const record = await nodeA.dht.get(`/ipns/${publish.name}`) + const value = await all(nodeA.dht.put(`/ipns/${publish.name}`, record, { verbose: true })) + expect(value).to.has.length(3) + expect(value[2].id.toString()).to.be.equal(nodeB.peerId.id) + expect(value[2].type).to.be.equal(5) }) }) } diff --git a/packages/interface-ipfs-core/src/pubsub/publish.js b/packages/interface-ipfs-core/src/pubsub/publish.js index f536f0c6bd..6145c0961d 100644 --- a/packages/interface-ipfs-core/src/pubsub/publish.js +++ b/packages/interface-ipfs-core/src/pubsub/publish.js @@ -4,7 +4,7 @@ const { Buffer } = require('buffer') const { nanoid } = require('nanoid') const { getTopic } = require('./utils') -const { getDescribe, getIt } = require('../utils/mocha') +const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ @@ -38,6 +38,11 @@ module.exports = (common, options) => { return ipfs.pubsub.publish(topic, 'hello friend') }) + it('should fail with undefined msg', async () => { + const topic = getTopic() + await expect(ipfs.pubsub.publish(topic)).to.eventually.rejectedWith('argument "data" is required') + }) + it('should publish message from buffer', () => { const topic = getTopic() return ipfs.pubsub.publish(topic, Buffer.from(nanoid())) diff --git a/packages/interface-ipfs-core/src/pubsub/subscribe.js b/packages/interface-ipfs-core/src/pubsub/subscribe.js index a13a7bdb61..08d9a7cb00 100644 --- a/packages/interface-ipfs-core/src/pubsub/subscribe.js +++ b/packages/interface-ipfs-core/src/pubsub/subscribe.js @@ -2,12 +2,14 @@ 'use strict' const { Buffer } = require('buffer') +const { nanoid } = require('nanoid') const pushable = require('it-pushable') const all = require('it-all') const { waitForPeers, getTopic } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const delay = require('delay') -const { isWebWorker } = require('ipfs-utils/src/env') +const AbortController = require('abort-controller') +const { isWebWorker, isNode } = require('ipfs-utils/src/env') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -163,6 +165,66 @@ module.exports = (common, options) => { return ipfs1.swarm.connect(ipfs2Addr) }) + it('should receive messages from a different node with floodsub', async function () { + if (!isNode) { + return this.skip() + } + const expectedString = 'should receive messages from a different node with floodsub' + const topic = `floodsub-${nanoid()}` + const ipfs1 = (await common.spawn({ + ipfsOptions: { + config: { + Pubsub: { + Router: 'floodsub' + } + } + } + })).api + const ipfs2 = (await common.spawn({ + type: isWebWorker ? 'go' : undefined, + ipfsOptions: { + config: { + Pubsub: { + Router: 'floodsub' + } + } + } + })).api + await ipfs1.swarm.connect(ipfs2.peerId.addresses[0]) + + const msgStream1 = pushable() + const msgStream2 = pushable() + + const sub1 = msg => { + msgStream1.push(msg) + msgStream1.end() + } + const sub2 = msg => { + msgStream2.push(msg) + msgStream2.end() + } + + const abort1 = new AbortController() + const abort2 = new AbortController() + await Promise.all([ + ipfs1.pubsub.subscribe(topic, sub1, { signal: abort1.signal }), + ipfs2.pubsub.subscribe(topic, sub2, { signal: abort2.signal }) + ]) + + await waitForPeers(ipfs2, topic, [ipfs1.peerId.id], 30000) + await ipfs2.pubsub.publish(topic, Buffer.from(expectedString)) + + const [sub1Msg] = await all(msgStream1) + expect(sub1Msg.data.toString()).to.be.eql(expectedString) + expect(sub1Msg.from).to.eql(ipfs2.peerId.id) + + const [sub2Msg] = await all(msgStream2) + expect(sub2Msg.data.toString()).to.be.eql(expectedString) + expect(sub2Msg.from).to.eql(ipfs2.peerId.id) + abort1.abort() + abort2.abort() + }) + it('should receive messages from a different node', async () => { const expectedString = 'hello from the other side' @@ -184,7 +246,7 @@ module.exports = (common, options) => { ]) await waitForPeers(ipfs2, topic, [ipfs1.peerId.id], 30000) - + await delay(5000) // gossipsub need this delay https://github.com/libp2p/go-libp2p-pubsub/issues/331 await ipfs2.pubsub.publish(topic, Buffer.from(expectedString)) const [sub1Msg] = await all(msgStream1) @@ -218,7 +280,7 @@ module.exports = (common, options) => { ]) await waitForPeers(ipfs2, topic, [ipfs1.peerId.id], 30000) - + await delay(5000) // gossipsub need this delay https://github.com/libp2p/go-libp2p-pubsub/issues/331 await ipfs2.pubsub.publish(topic, buffer) const [sub1Msg] = await all(msgStream1) @@ -256,7 +318,7 @@ module.exports = (common, options) => { ]) await waitForPeers(ipfs2, topic, [ipfs1.peerId.id], 30000) - + await delay(5000) // gossipsub need this delay https://github.com/libp2p/go-libp2p-pubsub/issues/331 outbox.forEach(msg => ipfs2.pubsub.publish(topic, Buffer.from(msg))) const sub1Msgs = await all(msgStream1) @@ -290,7 +352,7 @@ module.exports = (common, options) => { ]) await waitForPeers(ipfs1, topic, [ipfs2.peerId.id], 30000) - + await delay(5000) // gossipsub need this delay https://github.com/libp2p/go-libp2p-pubsub/issues/331 const startTime = new Date().getTime() for (let i = 0; i < count; i++) { diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 358de49501..54ca0a3f01 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -70,7 +70,7 @@ "devDependencies": { "aegir": "^22.0.0", "cross-env": "^7.0.0", - "go-ipfs-dep": "0.4.23-3", + "go-ipfs-dep": "^0.5.1", "interface-ipfs-core": "^0.135.1", "ipfsd-ctl": "^4.1.1", "it-all": "^1.0.1", diff --git a/packages/ipfs-http-client/src/dht/get.js b/packages/ipfs-http-client/src/dht/get.js index 4a99f5b82b..f1299e2097 100644 --- a/packages/ipfs-http-client/src/dht/get.js +++ b/packages/ipfs-http-client/src/dht/get.js @@ -1,22 +1,17 @@ 'use strict' const { Buffer } = require('buffer') -const encodeBufferURIComponent = require('../lib/encode-buffer-uri-component') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') const { Value } = require('./response-types') module.exports = configure(api => { return async function get (key, options = {}) { - if (!Buffer.isBuffer(key)) { - throw new Error('invalid key') - } - const res = await api.post('dht/get', { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - key: encodeBufferURIComponent(key), + arg: Buffer.isBuffer(key) ? key.toString() : key, ...options }), headers: options.headers @@ -24,7 +19,7 @@ module.exports = configure(api => { for await (const message of res.ndjson()) { if (message.Type === Value) { - return message.Extra + return Buffer.from(message.Extra, 'base64') } } diff --git a/packages/ipfs-http-client/src/dht/put.js b/packages/ipfs-http-client/src/dht/put.js index d345a9ee90..7110fc2bd2 100644 --- a/packages/ipfs-http-client/src/dht/put.js +++ b/packages/ipfs-http-client/src/dht/put.js @@ -5,6 +5,7 @@ const multiaddr = require('multiaddr') const toCamel = require('../lib/object-to-camel') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') +const multipartRequest = require('../lib/multipart-request') module.exports = configure(api => { return async function * put (key, value, options = {}) { @@ -12,13 +13,12 @@ module.exports = configure(api => { timeout: options.timeout, signal: options.signal, searchParams: toUrlSearchParams({ - arg: [ - key, - value - ], + arg: key, ...options }), - headers: options.headers + ...( + await multipartRequest(value, options.headers) + ) }) for await (let message of res.ndjson()) { diff --git a/packages/ipfs-http-client/src/files/ls.js b/packages/ipfs-http-client/src/files/ls.js index c9a3718729..2e22ddcdb5 100644 --- a/packages/ipfs-http-client/src/files/ls.js +++ b/packages/ipfs-http-client/src/files/ls.js @@ -17,12 +17,8 @@ module.exports = configure(api => { signal: options.signal, searchParams: toUrlSearchParams({ arg: CID.isCID(path) ? `/ipfs/${path}` : path, - - // TODO the args below are not in the go-ipfs or interface core docs - long: options.long == null ? true : options.long, - - // TODO: remove after go-ipfs 0.5 is released - l: options.long == null ? true : options.long, + // default long to true, diverges from go-ipfs where its false by default + long: true, ...options, stream: true }), diff --git a/packages/ipfs-http-client/src/lib/encode-buffer-uri-component.js b/packages/ipfs-http-client/src/lib/encode-buffer-uri-component.js deleted file mode 100644 index c231970cb2..0000000000 --- a/packages/ipfs-http-client/src/lib/encode-buffer-uri-component.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -// https://github.com/ipfs/js-ipfs-http-client/issues/569 -module.exports = function encodeBuffer (buf) { - let uriEncoded = '' - for (const byte of buf) { - // https://tools.ietf.org/html/rfc3986#page-14 - // ALPHA (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period (%2E), - // underscore (%5F), or tilde (%7E) - if ( - (byte >= 0x41 && byte <= 0x5A) || - (byte >= 0x61 && byte <= 0x7A) || - (byte >= 0x30 && byte <= 0x39) || - (byte === 0x2D) || - (byte === 0x2E) || - (byte === 0x5F) || - (byte === 0x7E) - ) { - uriEncoded += String.fromCharCode(byte) - } else { - uriEncoded += `%${byte.toString(16).padStart(2, '0')}` - } - } - return uriEncoded -} diff --git a/packages/ipfs-http-client/src/lib/multipart-request.js b/packages/ipfs-http-client/src/lib/multipart-request.js index 9ae1cbdf57..eee4e26b1b 100644 --- a/packages/ipfs-http-client/src/lib/multipart-request.js +++ b/packages/ipfs-http-client/src/lib/multipart-request.js @@ -7,7 +7,7 @@ const modeToString = require('../lib/mode-to-string') const mtimeToObject = require('../lib/mtime-to-object') const merge = require('merge-options').bind({ ignoreUndefined: true }) -async function multipartRequest (source, abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { +async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { async function * streamFiles (source) { try { let index = 0 diff --git a/packages/ipfs-http-client/src/pin/ls.js b/packages/ipfs-http-client/src/pin/ls.js index 4ff1606f53..5fdc17386d 100644 --- a/packages/ipfs-http-client/src/pin/ls.js +++ b/packages/ipfs-http-client/src/pin/ls.js @@ -25,14 +25,7 @@ module.exports = configure(api => { }) for await (const pin of res.ndjson()) { - if (pin.Keys) { // non-streaming response - // eslint-disable-next-line guard-for-in - for (const key in pin.Keys) { - yield { cid: new CID(key), type: pin.Keys[key].Type } - } - } else { - yield { cid: new CID(pin.Cid), type: pin.Type } - } + yield { cid: new CID(pin.Cid), type: pin.Type } } } }) diff --git a/packages/ipfs-http-client/src/pubsub/publish.js b/packages/ipfs-http-client/src/pubsub/publish.js index e7fe921ca7..0b24f4fc28 100644 --- a/packages/ipfs-http-client/src/pubsub/publish.js +++ b/packages/ipfs-http-client/src/pubsub/publish.js @@ -1,25 +1,29 @@ 'use strict' -const { Buffer } = require('buffer') -const encodeBuffer = require('../lib/encode-buffer-uri-component') const configure = require('../lib/configure') const toUrlSearchParams = require('../lib/to-url-search-params') +const multipartRequest = require('../lib/multipart-request') +const anySignal = require('any-signal') +const AbortController = require('abort-controller') module.exports = configure(api => { return async (topic, data, options = {}) => { - data = Buffer.from(data) - const searchParams = toUrlSearchParams({ - arg: [ - topic - ], + arg: topic, ...options }) - const res = await api.post(`pubsub/pub?${searchParams}&arg=${encodeBuffer(data)}`, { + // allow aborting requests on body errors + const controller = new AbortController() + const signal = anySignal([controller.signal, options.signal]) + + const res = await api.post('pubsub/pub', { timeout: options.timeout, - signal: options.signal, - headers: options.headers + signal, + searchParams, + ...( + await multipartRequest(data, controller, options.headers) + ) }) await res.text() diff --git a/packages/ipfs-http-client/test/interface.spec.js b/packages/ipfs-http-client/test/interface.spec.js index 62ea3258d4..f5be737c80 100644 --- a/packages/ipfs-http-client/test/interface.spec.js +++ b/packages/ipfs-http-client/test/interface.spec.js @@ -51,7 +51,12 @@ describe('interface-ipfs-core tests', () => { ] }) - tests.bitswap(commonFactory) + tests.bitswap(commonFactory, { + skip: [{ + name: 'should get the wantlist by peer ID for a different node', + reason: 'unskip when https://github.com/ipfs/go-bitswap/pull/390 is released in go-ipfs' + }] + }) tests.block(commonFactory, { skip: [{ diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index fa05fde163..d9e8d86bb9 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -71,7 +71,6 @@ "any-signal": "^1.1.0", "array-shuffle": "^1.0.1", "bignumber.js": "^9.0.0", - "binary-querystring": "^0.1.2", "bl": "^4.0.2", "bs58": "^4.0.1", "buffer": "^5.6.0", @@ -134,8 +133,8 @@ "libp2p-crypto": "^0.17.6", "libp2p-delegated-content-routing": "^0.4.4", "libp2p-delegated-peer-routing": "^0.4.2", - "libp2p-floodsub": "^0.20.0", - "libp2p-gossipsub": "^0.3.0", + "libp2p-floodsub": "^0.20.4", + "libp2p-gossipsub": "^0.3.1", "libp2p-kad-dht": "^0.18.7", "libp2p-keychain": "^0.6.0", "libp2p-mdns": "^0.13.1", @@ -181,9 +180,9 @@ "delay": "^4.3.0", "execa": "^4.0.0", "form-data": "^3.0.0", - "go-ipfs-dep": "0.4.23-3", + "go-ipfs-dep": "^0.5.1", "interface-ipfs-core": "^0.135.1", - "ipfs-interop": "^1.0.3", + "ipfs-interop": "^1.0.4", "ipfsd-ctl": "^4.1.1", "iso-random-stream": "^1.1.1", "it-first": "^1.0.1", diff --git a/packages/ipfs/src/cli/commands/files/ls.js b/packages/ipfs/src/cli/commands/files/ls.js index e374c14733..8248f205fb 100644 --- a/packages/ipfs/src/cli/commands/files/ls.js +++ b/packages/ipfs/src/cli/commands/files/ls.js @@ -1,6 +1,5 @@ 'use strict' -const all = require('it-all') const { asBoolean } = require('../../utils') @@ -21,13 +20,6 @@ module.exports = { coerce: asBoolean, describe: 'Use long listing format.' }, - sort: { - alias: 's', - type: 'boolean', - default: true, - coerce: asBoolean, - describe: 'Sort entries by name' - }, 'cid-base': { describe: 'CID base to use.' }, @@ -41,7 +33,6 @@ module.exports = { ctx: { ipfs, print }, path, long, - sort, cidBase, timeout }) { @@ -53,20 +44,6 @@ module.exports = { } } - // https://github.com/ipfs/go-ipfs/issues/5181 - if (sort) { - let files = await all(ipfs.files.ls(path || '/', { - timeout - })) - - files = files.sort((a, b) => { - return a.name.localeCompare(b.name) - }) - - files.forEach(printListing) - return - } - for await (const file of ipfs.files.ls(path || '/', { timeout })) { diff --git a/packages/ipfs/src/core/components/files/ls.js b/packages/ipfs/src/core/components/files/ls.js index 4aad5494da..d6dc7ba77a 100644 --- a/packages/ipfs/src/core/components/files/ls.js +++ b/packages/ipfs/src/core/components/files/ls.js @@ -1,17 +1,12 @@ 'use strict' const exporter = require('ipfs-unixfs-exporter') -const applyDefaultOptions = require('./utils/apply-default-options') const toMfsPath = require('./utils/to-mfs-path') const { MFS_FILE_TYPES, withTimeoutOption } = require('../../utils') -const defaultOptions = { - -} - const toOutput = (fsEntry) => { let type = 0 let size = fsEntry.node.size || fsEntry.node.length @@ -50,8 +45,6 @@ module.exports = (context) => { path = '/' } - options = applyDefaultOptions(options, defaultOptions) - const mfsPath = await toMfsPath(context, path) const fsDir = await exporter(mfsPath.mfsPath, context.ipld) diff --git a/packages/ipfs/src/core/components/key/gen.js b/packages/ipfs/src/core/components/key/gen.js index 86b6b0bd61..a9be31f5cb 100644 --- a/packages/ipfs/src/core/components/key/gen.js +++ b/packages/ipfs/src/core/components/key/gen.js @@ -5,6 +5,6 @@ const { withTimeoutOption } = require('../../utils') module.exports = ({ keychain }) => { return withTimeoutOption((name, options) => { options = options || {} - return keychain.createKey(name, options.type, options.size) + return keychain.createKey(name, options.type || 'rsa', options.size || 2048) }) } diff --git a/packages/ipfs/src/core/components/pubsub.js b/packages/ipfs/src/core/components/pubsub.js index 3d6dbcc9cc..71b43d4370 100644 --- a/packages/ipfs/src/core/components/pubsub.js +++ b/packages/ipfs/src/core/components/pubsub.js @@ -1,12 +1,18 @@ 'use strict' const { withTimeoutOption } = require('../utils') +const errCode = require('err-code') module.exports = ({ libp2p }) => { return { subscribe: withTimeoutOption((...args) => libp2p.pubsub.subscribe(...args)), unsubscribe: withTimeoutOption((...args) => libp2p.pubsub.unsubscribe(...args)), - publish: withTimeoutOption((...args) => libp2p.pubsub.publish(...args)), + publish: withTimeoutOption(async (topic, data, options) => { + if (!data) { + throw errCode(new Error('argument "data" is required'), 'ERR_ARG_REQUIRED') + } + await libp2p.pubsub.publish(topic, data) + }), ls: withTimeoutOption((...args) => libp2p.pubsub.getTopics(...args)), peers: withTimeoutOption((...args) => libp2p.pubsub.getSubscribers(...args)) } diff --git a/packages/ipfs/src/http/api/resources/files/ls.js b/packages/ipfs/src/http/api/resources/files/ls.js index 3b7de05733..75b642af26 100644 --- a/packages/ipfs/src/http/api/resources/files/ls.js +++ b/packages/ipfs/src/http/api/resources/files/ls.js @@ -49,14 +49,6 @@ const mfsLs = { override: true, ignoreUndefined: true }) - .rename('l', 'long', { - override: true, - ignoreUndefined: true - }) - .rename('s', 'stream', { - override: true, - ignoreUndefined: true - }) } }, async handler (request, h) { diff --git a/packages/ipfs/src/http/api/resources/pubsub.js b/packages/ipfs/src/http/api/resources/pubsub.js index ccd1287075..4ae6c2ca56 100644 --- a/packages/ipfs/src/http/api/resources/pubsub.js +++ b/packages/ipfs/src/http/api/resources/pubsub.js @@ -3,7 +3,8 @@ const Joi = require('../../utils/joi') const PassThrough = require('stream').PassThrough const bs58 = require('bs58') -const binaryQueryString = require('binary-querystring') +const all = require('it-all') +const multipart = require('../../utils/multipart-request-parser') const Boom = require('@hapi/boom') exports.subscribe = { @@ -77,20 +78,43 @@ exports.subscribe = { exports.publish = { options: { + payload: { + parse: false, + output: 'stream' + }, + pre: [{ + assign: 'data', + method: async (request, h) => { + if (!request.payload) { + throw Boom.badRequest('argument "data" is required') + } + + let data + + for await (const part of multipart(request)) { + if (part.type === 'file') { + data = Buffer.concat(await all(part.content)) + } + } + + if (!data || data.byteLength === 0) { + throw Boom.badRequest('argument "data" is required') + } + + return data + } + }], validate: { options: { allowUnknown: true, stripUnknown: true }, query: Joi.object().keys({ - args: Joi.array().ordered( - Joi.string().required(), - Joi.binary().min(1).required() - ).required(), + topic: Joi.string().required(), discover: Joi.boolean(), timeout: Joi.timeout() }) - .rename('arg', 'args', { + .rename('arg', 'topic', { override: true, ignoreUndefined: true }) @@ -106,19 +130,17 @@ exports.publish = { ipfs } }, + pre: { + data + }, query: { - args: [ - topic - ], + topic, timeout } } = request - const rawArgs = binaryQueryString(request.url.search) - const buf = rawArgs.arg && rawArgs.arg[1] - try { - await ipfs.pubsub.publish(topic, buf, { + await ipfs.pubsub.publish(topic, data, { signal, timeout }) diff --git a/packages/ipfs/test/cli/files/ls.js b/packages/ipfs/test/cli/files/ls.js index a24b859b66..6ed54704da 100644 --- a/packages/ipfs/test/cli/files/ls.js +++ b/packages/ipfs/test/cli/files/ls.js @@ -143,90 +143,6 @@ describe('ls', () => { expect(output).to.include(files[0].size) }) - it('should list a path without sorting', async () => { - const files = [{ - cid: fileCid, - name: 'file-name', - size: 'file-size', - mode: 0o755, - mtime: { - secs: Date.now() / 1000, - nsecs: 0 - } - }] - - ipfs.files.ls = sinon.stub().withArgs('/foo', defaultOptions).returns(files) - - await cli('files ls --sort false /foo', { ipfs, print }) - - expect(ipfs.files.ls.callCount).to.equal(1) - expect(output).to.include(files[0].name) - }) - - it('should list a path without sorting (short option)', async () => { - const files = [{ - cid: fileCid, - name: 'file-name', - size: 'file-size', - mode: 0o755, - mtime: { - secs: Date.now() / 1000, - nsecs: 0 - } - }] - - ipfs.files.ls = sinon.stub().withArgs('/foo', defaultOptions).returns(files) - - await cli('files ls -s false /foo', { ipfs, print }) - - expect(ipfs.files.ls.callCount).to.equal(1) - expect(output).to.include(files[0].name) - }) - - it('should list a path with details without sorting', async () => { - const files = [{ - cid: fileCid, - name: 'file-name', - size: 'file-size', - mode: 0o755, - mtime: { - secs: Date.now() / 1000, - nsecs: 0 - } - }] - - ipfs.files.ls = sinon.stub().withArgs('/foo', defaultOptions).returns(files) - - await cli('files ls --long --sort false /foo', { ipfs, print }) - - expect(ipfs.files.ls.callCount).to.equal(1) - expect(output).to.include(files[0].cid.toString()) - expect(output).to.include(files[0].name) - expect(output).to.include(files[0].size) - }) - - it('should list a path with details without sorting (short option)', async () => { - const files = [{ - cid: fileCid, - name: 'file-name', - size: 'file-size', - mode: 0o755, - mtime: { - secs: Date.now() / 1000, - nsecs: 0 - } - }] - - ipfs.files.ls = sinon.stub().withArgs('/foo', defaultOptions).returns(files) - - await cli('files ls -l -s false /foo', { ipfs, print }) - - expect(ipfs.files.ls.callCount).to.equal(1) - expect(output).to.include(files[0].cid.toString()) - expect(output).to.include(files[0].name) - expect(output).to.include(files[0].size) - }) - it('should list a path with a timeout', async () => { const path = '/foo' diff --git a/packages/ipfs/test/http-api/inject/pubsub.js b/packages/ipfs/test/http-api/inject/pubsub.js index af5bb21374..4b15d5ef66 100644 --- a/packages/ipfs/test/http-api/inject/pubsub.js +++ b/packages/ipfs/test/http-api/inject/pubsub.js @@ -5,32 +5,22 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const testHttpMethod = require('../../utils/test-http-method') const http = require('../../utils/http') +const FormData = require('form-data') const sinon = require('sinon') const { AbortSignal } = require('abort-controller') const randomBytes = require('iso-random-stream/src/random') const { Buffer } = require('buffer') - -function encodeBuffer (buf) { - let uriEncoded = '' - for (const byte of buf) { - // https://tools.ietf.org/html/rfc3986#page-14 - // ALPHA (%41-%5A and %61-%7A), DIGIT (%30-%39), hyphen (%2D), period (%2E), - // underscore (%5F), or tilde (%7E) - if ( - (byte >= 0x41 && byte <= 0x5A) || - (byte >= 0x61 && byte <= 0x7A) || - (byte >= 0x30 && byte <= 0x39) || - (byte === 0x2D) || - (byte === 0x2E) || - (byte === 0x5F) || - (byte === 0x7E) - ) { - uriEncoded += String.fromCharCode(byte) - } else { - uriEncoded += `%${byte.toString(16).padStart(2, '0')}` - } +const streamToPromise = require('stream-to-promise') + +const sendData = async (data) => { + const form = new FormData() + form.append('data', data) + const headers = form.getHeaders() + const payload = await streamToPromise(form) + return { + headers, + payload } - return uriEncoded } describe('/pubsub', () => { @@ -104,7 +94,8 @@ describe('/pubsub', () => { it('returns 200 with topic and buffer', async () => { const res = await http({ method: 'POST', - url: `/api/v0/pubsub/pub?arg=${topic}&arg=${buf}` + url: `/api/v0/pubsub/pub?arg=${topic}`, + ...await sendData(buf) }, { ipfs }) expect(res).to.have.property('statusCode', 200) @@ -115,7 +106,8 @@ describe('/pubsub', () => { const buf = randomBytes(10) const res = await http({ method: 'POST', - url: `/api/v0/pubsub/pub?arg=${topic}&arg=${encodeBuffer(buf)}` + url: `/api/v0/pubsub/pub?arg=${topic}`, + ...await sendData(buf) }, { ipfs }) expect(res).to.have.property('statusCode', 200) @@ -125,7 +117,8 @@ describe('/pubsub', () => { it('returns 400 with topic and empty buffer', async () => { const res = await http({ method: 'POST', - url: `/api/v0/pubsub/pub?arg=${topic}&arg=${encodeBuffer([])}` + url: `/api/v0/pubsub/pub?arg=${topic}`, + ...await sendData(Buffer.from('')) }, { ipfs }) expect(res).to.have.property('statusCode', 400) @@ -134,7 +127,8 @@ describe('/pubsub', () => { it('accepts a timeout', async () => { const res = await http({ method: 'POST', - url: `/api/v0/pubsub/pub?arg=${topic}&arg=${buf}&timeout=1s` + url: `/api/v0/pubsub/pub?arg=${topic}&timeout=1s`, + ...await sendData(buf) }, { ipfs }) expect(res).to.have.property('statusCode', 200) diff --git a/packages/ipfs/test/http-api/interface.js b/packages/ipfs/test/http-api/interface.js index 5554ea25c6..2797b0b98c 100644 --- a/packages/ipfs/test/http-api/interface.js +++ b/packages/ipfs/test/http-api/interface.js @@ -6,6 +6,7 @@ const factory = require('../utils/factory') const { isNode, isBrowser } = require('ipfs-utils/src/env') /** @typedef { import("ipfsd-ctl").ControllerOptions } ControllerOptions */ + describe('interface-ipfs-core over ipfs-http-client tests', function () { this.timeout(20000) @@ -112,13 +113,17 @@ describe('interface-ipfs-core over ipfs-http-client tests', function () { }] }) - tests.pubsub(factory({ - type: 'js', - ipfsBin: './src/cli/bin.js', - go: { - args: ['--enable-pubsub-experiment'] + tests.pubsub(factory( + { + type: 'js', + ipfsBin: './src/cli/bin.js' + }, + { + go: { + args: ['--enable-pubsub-experiment'] + } } - })) + )) tests.repo(commonFactory) diff --git a/packages/ipfs/test/http-api/routes.js b/packages/ipfs/test/http-api/routes.js index ca9db8ca37..126d8e7b38 100644 --- a/packages/ipfs/test/http-api/routes.js +++ b/packages/ipfs/test/http-api/routes.js @@ -12,7 +12,17 @@ require('./inject/dns') require('./inject/files') require('./inject/id') require('./inject/key') -require('./inject/mfs') +require('./inject/mfs/chmod') +require('./inject/mfs/cp') +require('./inject/mfs/flush') +require('./inject/mfs/ls') +require('./inject/mfs/mkdir') +require('./inject/mfs/mv') +require('./inject/mfs/read') +require('./inject/mfs/rm') +require('./inject/mfs/stat') +require('./inject/mfs/touch') +require('./inject/mfs/write') require('./inject/name') require('./inject/object') require('./inject/pin')