diff --git a/package.json b/package.json index a092ea0c..2a1febff 100644 --- a/package.json +++ b/package.json @@ -81,12 +81,14 @@ "ipfs-repo-migrations": "^7.0.1", "ipfs-utils": "^6.0.0", "ipld-block": "^0.11.0", + "it-filter": "^1.0.2", "it-map": "^1.0.2", "it-pushable": "^1.4.0", "just-safe-get": "^2.0.0", "just-safe-set": "^2.1.0", "merge-options": "^3.0.4", "multibase": "^4.0.1", + "multihashes": "^4.0.2", "p-queue": "^6.0.0", "proper-lockfile": "^4.0.0", "sort-keys": "^4.0.0", diff --git a/src/blockstore.js b/src/blockstore.js index 3aad0952..cf52aa3b 100644 --- a/src/blockstore.js +++ b/src/blockstore.js @@ -11,6 +11,7 @@ const pushable = require('it-pushable') * @typedef {import("interface-datastore").Datastore} Datastore * @typedef {import("interface-datastore").Options} DatastoreOptions * @typedef {import("cids")} CID + * @typedef {import('./types').Blockstore} Blockstore */ /** @@ -36,19 +37,14 @@ function maybeWithSharding (filestore, options) { /** * @param {Datastore | ShardingDatastore} store + * @returns {Blockstore} */ function createBaseStore (store) { return { open () { return store.open() }, - /** - * Query the store - * - * @param {Query} query - * @param {DatastoreOptions} [options] - * @returns {AsyncIterable} - */ + async * query (query, options) { for await (const { key, value } of store.query(query, options)) { // TODO: we should make this a different method @@ -61,13 +57,6 @@ function createBaseStore (store) { } }, - /** - * Get a single block by CID - * - * @param {CID} cid - * @param {DatastoreOptions} [options] - * @returns {Promise} - */ async get (cid, options) { const key = cidToKey(cid) const blockData = await store.get(key, options) @@ -75,26 +64,12 @@ function createBaseStore (store) { return new Block(blockData, cid) }, - /** - * Like get, but for more - * - * @param {Iterable | AsyncIterable} cids - * @param {DatastoreOptions} [options] - * @returns {AsyncIterable} - */ async * getMany (cids, options) { for await (const cid of cids) { yield this.get(cid, options) } }, - /** - * Write a single block to the store - * - * @param {Block} block - * @param {DatastoreOptions} [options] - * @returns {Promise} - */ async put (block, options) { if (!Block.isBlock(block)) { throw new Error('invalid block') @@ -110,13 +85,6 @@ function createBaseStore (store) { return block }, - /** - * Like put, but for more - * - * @param {AsyncIterable|Iterable} blocks - * @param {DatastoreOptions} [options] - * @returns {AsyncIterable} - */ async * putMany (blocks, options) { // eslint-disable-line require-await // we cannot simply chain to `store.putMany` because we convert a CID into // a key based on the multihash only, so we lose the version & codec and @@ -158,41 +126,18 @@ function createBaseStore (store) { yield * output }, - /** - * Does the store contain block with this CID? - * - * @param {CID} cid - * @param {DatastoreOptions} [options] - */ has (cid, options) { return store.has(cidToKey(cid), options) }, - /** - * Delete a block from the store - * - * @param {CID} cid - * @param {DatastoreOptions} [options] - * @returns {Promise} - */ delete (cid, options) { return store.delete(cidToKey(cid), options) }, - /** - * Delete a block from the store - * - * @param {AsyncIterable | Iterable} cids - * @param {DatastoreOptions} [options] - */ deleteMany (cids, options) { return store.deleteMany(map(cids, cid => cidToKey(cid)), options) }, - /** - * Close the store - * - */ close () { return store.close() } diff --git a/src/config.js b/src/config.js index 3b1a9c38..83657b02 100644 --- a/src/config.js +++ b/src/config.js @@ -139,7 +139,9 @@ module.exports = (store) => { const value = m.value if (key) { const config = await configStore.get() - _set(config, key, value) + if (typeof config === 'object' && config !== null) { + _set(config, key, value) + } return _saveAll(config) } return _saveAll(value) diff --git a/src/idstore.js b/src/idstore.js new file mode 100644 index 00000000..cbcfd940 --- /dev/null +++ b/src/idstore.js @@ -0,0 +1,138 @@ +'use strict' + +const Block = require('ipld-block') +const filter = require('it-filter') +const mh = require('multihashes') +const pushable = require('it-pushable') +const drain = require('it-drain') +const CID = require('cids') +const errcode = require('err-code') + +/** + * @typedef {import("interface-datastore").Query} Query + * @typedef {import("interface-datastore").Datastore} Datastore + * @typedef {import("interface-datastore").Options} DatastoreOptions + * @typedef {import('./types').Blockstore} Blockstore + */ + +/** + * + * @param {Blockstore} blockstore + */ +module.exports = createIdStore + +/** + * @param {Blockstore} store + * @returns {Blockstore} + */ +function createIdStore (store) { + return { + open () { + return store.open() + }, + query (query, options) { + return store.query(query, options) + }, + + async get (cid, options) { + const extracted = extractContents(cid) + if (extracted.isIdentity) { + return Promise.resolve(new Block(extracted.digest, cid)) + } + return store.get(cid, options) + }, + + async * getMany (cids, options) { + for await (const cid of cids) { + yield this.get(cid, options) + } + }, + + async put (block, options) { + const { isIdentity } = extractContents(block.cid) + if (isIdentity) { + return Promise.resolve(block) + } + return store.put(block, options) + }, + + async * putMany (blocks, options) { + // in order to return all blocks. we're going to assemble a seperate iterable + // return rather than return the resolves of store.putMany using the same + // process used by blockstore.putMany + const output = pushable() + + // process.nextTick runs on the microtask queue, setImmediate runs on the next + // event loop iteration so is slower. Use process.nextTick if it is available. + const runner = process && process.nextTick ? process.nextTick : setImmediate + + runner(async () => { + try { + await drain(store.putMany(async function * () { + for await (const block of blocks) { + if (!extractContents(block.cid).isIdentity) { + yield block + } + // if non identity blocks successfully write, blocks are included in output + output.push(block) + } + }())) + + output.end() + } catch (err) { + output.end(err) + } + }) + + yield * output + }, + + has (cid, options) { + const { isIdentity } = extractContents(cid) + if (isIdentity) { + return Promise.resolve(true) + } + return store.has(cid, options) + }, + + delete (cid, options) { + const { isIdentity } = extractContents(cid) + if (isIdentity) { + return Promise.resolve() + } + return store.delete(cid, options) + }, + + deleteMany (cids, options) { + return store.deleteMany(filter(cids, (cid) => !extractContents(cid).isIdentity), options) + }, + + close () { + return store.close() + } + } +} + +/** + * @param {CID} k + * @returns {{ isIdentity: false } | { isIdentity: true, digest: Uint8Array}} + */ +function extractContents (k) { + if (!CID.isCID(k)) { + throw errcode(new Error('Not a valid cid'), 'ERR_INVALID_CID') + } + + // Pre-check by calling Prefix(), this much faster than extracting the hash. + const decoded = mh.decode(k.multihash) + + if (decoded.name !== 'identity') { + return { + isIdentity: false + } + } + + return { + isIdentity: true, + digest: decoded.digest + } +} diff --git a/src/index.js b/src/index.js index 7a152fbd..d625bf9c 100644 --- a/src/index.js +++ b/src/index.js @@ -16,6 +16,7 @@ const config = require('./config') const spec = require('./spec') const apiAddr = require('./api-addr') const blockstore = require('./blockstore') +const idstore = require('./idstore') const defaultOptions = require('./default-options') const defaultDatastore = require('./default-datastore') const ERRORS = require('./errors') @@ -66,8 +67,8 @@ class IpfsRepo { this.keys = backends.create('keys', pathJoin(this.path, 'keys'), this.options) this.pins = backends.create('pins', pathJoin(this.path, 'pins'), this.options) const blocksBaseStore = backends.create('blocks', pathJoin(this.path, 'blocks'), this.options) - this.blocks = blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks) - + const blockStore = blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks) + this.blocks = idstore(blockStore) this.version = version(this.root) this.config = config(this.root) this.spec = spec(this.root) @@ -437,7 +438,7 @@ module.exports.errors = ERRORS * @param {any} _config */ function buildConfig (_config) { - _config.datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore', {})) + _config.datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore')) return _config } @@ -449,7 +450,7 @@ function buildDatastoreSpec (_config) { /** @type { {type: string, mounts: Array<{mountpoint: string, type: string, prefix: string, child: {type: string, path: 'string', sync: boolean, shardFunc: string}}>}} */ const spec = { ...defaultDatastore.Spec, - ..._get(_config, 'datastore.Spec', {}) + ..._get(_config, 'datastore.Spec') } return { diff --git a/src/types.d.ts b/src/types.d.ts index 3078bc63..e81753de 100644 --- a/src/types.d.ts +++ b/src/types.d.ts @@ -1,6 +1,8 @@ -import type { Datastore } from 'interface-datastore' +import type { Datastore, Options as DatastoreOptions, Query } from 'interface-datastore' import type { BigNumber } from 'bignumber.js' +import type CID from 'cids' + export type AwaitIterable = Iterable | AsyncIterable export type Await = Promise | T @@ -56,3 +58,56 @@ export interface Stat { numObjects: BigNumber repoSize: BigNumber } + +export interface Block { + cid: CID + data: Uint8Array +} + +export interface Blockstore { + open: () => Promise + /** + * Query the store + */ + query: (query: Query, options?: DatastoreOptions) => AsyncIterable + + /** + * Get a single block by CID + */ + get: (cid: CID, options?: DatastoreOptions) => Promise + + /** + * Like get, but for more + */ + getMany: (cids: AwaitIterable, options?: DatastoreOptions) => AsyncIterable + + /** + * Write a single block to the store + */ + put: (block: Block, options?: DatastoreOptions) => Promise + + /** + * Like put, but for more + */ + putMany: (blocks: AwaitIterable, options?: DatastoreOptions) => AsyncIterable + + /** + * Does the store contain block with this CID? + */ + has: (cid: CID, options?: DatastoreOptions) => Promise + + /** + * Delete a block from the store + */ + delete: (cid: CID, options?: DatastoreOptions) => Promise + + /** + * Delete a block from the store + */ + deleteMany: (cids: AwaitIterable, options?: DatastoreOptions) => AsyncIterable + + /** + * Close the store + */ + close: () => Promise +} \ No newline at end of file diff --git a/test/blockstore-test.js b/test/blockstore-test.js index 6962ca7a..0838b862 100644 --- a/test/blockstore-test.js +++ b/test/blockstore-test.js @@ -36,12 +36,17 @@ module.exports = (repo) => { describe('blockstore', () => { const blockData = range(100).map((i) => uint8ArrayFromString(`hello-${i}-${Math.random()}`)) const bData = uint8ArrayFromString('hello world') + const identityData = uint8ArrayFromString('A16461736466190144', 'base16upper') + /** @type {Block} */ let b - + /** @type {CID} */ + let identityCID before(async () => { const hash = await multihashing(bData, 'sha2-256') b = new Block(bData, new CID(hash)) + const identityHash = await multihashing(identityData, 'identity') + identityCID = new CID(1, 'dag-cbor', identityHash) }) describe('.put', () => { @@ -58,6 +63,16 @@ module.exports = (repo) => { await repo.blocks.put(b) }) + it('does not write an identity block', async () => { + const identityBlock = new Block(identityData, identityCID) + await repo.blocks.put(identityBlock) + const cids = await all(repo.blocks.query({ + keysOnly: true + })) + const rawCID = new CID(1, 'raw', identityCID.multihash) + expect(cids).to.not.deep.include(rawCID) + }) + it('multi write (locks)', async () => { await Promise.all([repo.blocks.put(b), repo.blocks.put(b)]) }) @@ -95,6 +110,19 @@ module.exports = (repo) => { } }) + it('.putMany with identity block included', async function () { + const d = uint8ArrayFromString('many' + Math.random()) + const hash = await multihashing(d, 'sha2-256') + const blocks = [new Block(d, new CID(1, 'raw', hash)), new Block(identityData, identityCID)] + const put = await all(repo.blocks.putMany(blocks)) + expect(put).to.deep.equal(blocks) + const cids = await all(repo.blocks.query({ + keysOnly: true + })) + expect(cids).to.deep.include(new CID(1, 'raw', hash)) + expect(cids).to.not.deep.include(new CID(1, 'raw', identityCID.multihash)) + }) + it('returns an error on invalid block', () => { // @ts-expect-error return expect(repo.blocks.put('hello')).to.eventually.be.rejected() @@ -206,6 +234,11 @@ module.exports = (repo) => { expect(err2).to.deep.equal(err) } }) + + it('can load an identity hash without storing first', async () => { + const block = await repo.blocks.get(identityCID) + expect(block.data).to.be.eql(identityData) + }) }) describe('.getMany', () => { @@ -223,6 +256,12 @@ module.exports = (repo) => { expect(blocks).to.deep.include(b) }) + it('including a block with identity has', async () => { + const blocks = await all(repo.blocks.getMany([b.cid, identityCID])) + expect(blocks).to.deep.include(b) + expect(blocks).to.deep.include(new Block(identityData, identityCID)) + }) + it('massive read', async function () { this.timeout(15000) // add time for ci const num = 20 * 100 @@ -340,6 +379,11 @@ module.exports = (repo) => { expect(exists).to.eql(true) }) + it('identity hash block, not written to store', async () => { + const exists = await repo.blocks.has(identityCID) + expect(exists).to.eql(true) + }) + it('non existent block', async () => { const exists = await repo.blocks.has(new CID('QmbcpFjzamCj5ZZdduW32ctWUPvbGMwQZk2ghWK6PrKswE')) expect(exists).to.eql(false) @@ -386,6 +430,12 @@ module.exports = (repo) => { expect(exists).to.equal(false) }) + it('identity cid does nothing', async () => { + await repo.blocks.delete(identityCID) + const exists = await repo.blocks.has(identityCID) + expect(exists).to.equal(true) + }) + it('throws when passed an invalid cid', () => { // @ts-expect-error return expect(() => repo.blocks.delete('foo')).to.throw().with.property('code', 'ERR_INVALID_CID') @@ -399,6 +449,14 @@ module.exports = (repo) => { expect(exists).to.equal(false) }) + it('including identity cid', async () => { + await drain(repo.blocks.deleteMany([b.cid, identityCID])) + const exists = await repo.blocks.has(b.cid) + expect(exists).to.equal(false) + const identityExists = await repo.blocks.has(identityCID) + expect(identityExists).to.equal(true) + }) + it('throws when passed an invalid cid', () => { return expect(drain(repo.blocks.deleteMany(['foo']))).to.eventually.be.rejected().with.property('code', 'ERR_INVALID_CID') })