diff --git a/package.json b/package.json index 810b1b2..ff5168f 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,11 @@ }, "dependencies": { "debug": "^4.1.1", - "interface-datastore": "^3.0.1" + "interface-datastore": "ipfs/interface-datastore#feat/split-query-into-query-and-query-keys", + "it-filter": "^1.0.2", + "it-map": "^1.0.5", + "it-merge": "^1.0.1", + "it-take": "^1.0.1" }, "engines": { "node": ">=12.0.0" diff --git a/src/keytransform.js b/src/keytransform.js index cefc301..9ba7765 100644 --- a/src/keytransform.js +++ b/src/keytransform.js @@ -1,12 +1,14 @@ 'use strict' -const { Adapter, utils } = require('interface-datastore') -const map = utils.map +const { Adapter } = require('interface-datastore') +const map = require('it-map') + /** * @typedef {import('interface-datastore').Datastore} Datastore * @typedef {import('interface-datastore').Options} Options * @typedef {import('interface-datastore').Batch} Batch * @typedef {import('interface-datastore').Query} Query + * @typedef {import('interface-datastore').KeyQuery} KeyQuery * @typedef {import('interface-datastore').Key} Key * @typedef {import('./types').KeyTransform} KeyTransform */ @@ -90,9 +92,21 @@ class KeyTransformDatastore extends Adapter { * @param {Options} [options] */ query (q, options) { - return map(this.child.query(q, options), e => { - e.key = this.transform.invert(e.key) - return e + return map(this.child.query(q, options), ({ key, value }) => { + return { + key: this.transform.invert(key), + value + } + }) + } + + /** + * @param {KeyQuery} q + * @param {Options} [options] + */ + queryKeys (q, options) { + return map(this.child.queryKeys(q, options), key => { + return this.transform.invert(key) }) } diff --git a/src/mount.js b/src/mount.js index 261ce98..a5ff3a2 100644 --- a/src/mount.js +++ b/src/mount.js @@ -3,12 +3,13 @@ const { Adapter, Key, Errors, utils: { - filter, - take, sortAll, replaceStartWith } } = require('interface-datastore') +const filter = require('it-filter') +const take = require('it-take') +const merge = require('it-merge') const Keytransform = require('./keytransform') @@ -17,7 +18,8 @@ const Keytransform = require('./keytransform') * @typedef {import('interface-datastore').Options} Options * @typedef {import('interface-datastore').Batch} Batch * @typedef {import('interface-datastore').Query} Query - * @typedef {import('interface-datastore').Pair} Pair + * @typedef {import('interface-datastore').KeyQuery} KeyQuery + * @typedef {import('./types').KeyTransform} KeyTransform */ /** @@ -27,13 +29,12 @@ const Keytransform = require('./keytransform') /** * A datastore that can combine multiple stores inside various - * key prefixs. + * key prefixes * * @implements {Datastore} */ class MountDatastore extends Adapter { /** - * * @param {Array<{prefix: Key, datastore: Datastore}>} mounts */ constructor (mounts) { @@ -47,7 +48,7 @@ class MountDatastore extends Adapter { } /** - * Lookup the matching datastore for the given key. + * Lookup the matching datastore for the given key * * @private * @param {Key} key @@ -186,12 +187,11 @@ class MountDatastore extends Adapter { return ks.query({ prefix: prefix, - filters: q.filters, - keysOnly: q.keysOnly + filters: q.filters }, options) }) - let it = _many(qs) + let it = merge(...qs) if (q.filters) q.filters.forEach(f => { it = filter(it, f) }) if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) }) if (q.offset != null) { @@ -202,20 +202,44 @@ class MountDatastore extends Adapter { return it } -} -/** - * @param {ArrayLike>} iterable - * @returns {AsyncIterable} - */ -function _many (iterable) { - return (async function * () { - for (let i = 0; i < iterable.length; i++) { - for await (const v of iterable[i]) { - yield v + /** + * @param {KeyQuery} q + * @param {Options} [options] + */ + queryKeys (q, options) { + const qs = this.mounts.map(m => { + const ks = new Keytransform(m.datastore, { + convert: (key) => { + throw new Error('should never be called') + }, + invert: (key) => { + return m.prefix.child(key) + } + }) + + let prefix + if (q.prefix != null) { + prefix = replaceStartWith(q.prefix, m.prefix.toString()) } + + return ks.queryKeys({ + prefix: prefix, + filters: q.filters + }, options) + }) + + let it = merge(...qs) + if (q.filters) q.filters.forEach(f => { it = filter(it, f) }) + if (q.orders) q.orders.forEach(o => { it = sortAll(it, o) }) + if (q.offset != null) { + let i = 0 + it = filter(it, () => i++ >= /** @type {number} */ (q.offset)) } - })() + if (q.limit != null) it = take(it, q.limit) + + return it + } } module.exports = MountDatastore diff --git a/src/namespace.js b/src/namespace.js index 40dab31..3a65565 100644 --- a/src/namespace.js +++ b/src/namespace.js @@ -5,6 +5,7 @@ const KeytransformDatastore = require('./keytransform') /** * @typedef {import('interface-datastore').Datastore} Datastore * @typedef {import('interface-datastore').Query} Query + * @typedef {import('interface-datastore').KeyQuery} KeyQuery * @typedef {import('interface-datastore').Options} Options * @typedef {import('interface-datastore').Batch} Batch * @typedef {import('./types').KeyTransform} KeyTransform @@ -57,6 +58,19 @@ class NamespaceDatastore extends KeytransformDatastore { } return super.query(q, options) } + + /** + * @param {KeyQuery} q + * @param {Options} [options] + */ + queryKeys (q, options) { + if (q.prefix && this.prefix.toString() !== '/') { + return super.queryKeys(Object.assign({}, q, { + prefix: this.prefix.child(new Key(q.prefix)).toString() + })) + } + return super.queryKeys(q, options) + } } module.exports = NamespaceDatastore diff --git a/src/shard.js b/src/shard.js index 4b95edb..8d73fad 100644 --- a/src/shard.js +++ b/src/shard.js @@ -1,6 +1,6 @@ 'use strict' -const { Key, utils: { utf8Decoder } } = require('interface-datastore') +const { Key } = require('interface-datastore') const readme = require('./shard-readme') /** @@ -149,7 +149,7 @@ const readShardFun = async (path, store) => { // @ts-ignore const get = typeof store.getRaw === 'function' ? store.getRaw.bind(store) : store.get.bind(store) const res = await get(key) - return parseShardFun(utf8Decoder.decode(res || '').trim()) + return parseShardFun(new TextDecoder().decode(res || '').trim()) } module.exports = { diff --git a/src/sharding.js b/src/sharding.js index 15b5715..a493556 100644 --- a/src/sharding.js +++ b/src/sharding.js @@ -1,6 +1,6 @@ 'use strict' -const { Adapter, Key, utils: { utf8Encoder }, Errors } = require('interface-datastore') +const { Adapter, Key, Errors } = require('interface-datastore') const sh = require('./shard') const KeytransformStore = require('./keytransform') @@ -11,6 +11,11 @@ const shardReadmeKey = new Key(sh.README_FN) * @typedef {import('interface-datastore').Options} Options * @typedef {import('interface-datastore').Batch} Batch * @typedef {import('interface-datastore').Query} Query + * @typedef {import('interface-datastore').QueryFilter} QueryFilter + * @typedef {import('interface-datastore').QueryOrder} QueryOrder + * @typedef {import('interface-datastore').KeyQuery} KeyQuery + * @typedef {import('interface-datastore').KeyQueryFilter} KeyQueryFilter + * @typedef {import('interface-datastore').KeyQueryOrder} KeyQueryOrder * @typedef {import('interface-datastore').Pair} Pair * @typedef {import('./types').Shard} Shard * @@ -107,8 +112,8 @@ class ShardingDatastore extends Adapter { // @ts-ignore i have no idea what putRaw is or saw any implementation const put = typeof store.putRaw === 'function' ? store.putRaw.bind(store) : store.put.bind(store) await Promise.all([ - put(shardKey, utf8Encoder.encode(shard.toString() + '\n')), - put(shardReadmeKey, utf8Encoder.encode(sh.readme)) + put(shardKey, new TextEncoder().encode(shard.toString() + '\n')), + put(shardReadmeKey, new TextEncoder().encode(sh.readme)) ]) return shard @@ -167,15 +172,15 @@ class ShardingDatastore extends Adapter { */ query (q, options) { const tq = { - keysOnly: q.keysOnly, offset: q.offset, limit: q.limit, - /** @type Array<(items: Pair[]) => Await> */ + /** @type {QueryOrder[]} */ orders: [], + /** @type {QueryFilter[]} */ filters: [ - /** @type {(item: Pair) => boolean} */ + /** @type {QueryFilter} */ e => e.key.toString() !== shardKey.toString(), - /** @type {(item: Pair) => boolean} */ + /** @type {QueryFilter} */ e => e.key.toString() !== shardReadmeKey.toString() ] } @@ -188,27 +193,90 @@ class ShardingDatastore extends Adapter { } if (q.filters != null) { - // @ts-ignore - can't find a way to easily type this - const filters = q.filters.map((f) => (e) => { - return f(Object.assign({}, e, { - key: this._invertKey(e.key) - })) + const filters = q.filters.map(f => { + /** @type {QueryFilter} */ + const filter = ({ key, value }) => { + return f({ + key: this._invertKey(key), + value + }) + } + + return filter }) tq.filters = tq.filters.concat(filters) } if (q.orders != null) { - tq.orders = q.orders.map((o) => async (res) => { - res.forEach((e) => { e.key = this._invertKey(e.key) }) - const ordered = await o(res) - ordered.forEach((e) => { e.key = this._convertKey(e.key) }) - return ordered + tq.orders = q.orders.map(o => { + /** @type {QueryOrder} */ + const order = (a, b) => { + return o({ + key: this._invertKey(a.key), + value: a.value + }, { + key: this._invertKey(b.key), + value: b.value + }) + } + + return order }) } return this.child.query(tq, options) } + /** + * @param {KeyQuery} q + * @param {Options} [options] + */ + queryKeys (q, options) { + const tq = { + offset: q.offset, + limit: q.limit, + /** @type {KeyQueryOrder[]} */ + orders: [], + /** @type {KeyQueryFilter[]} */ + filters: [ + /** @type {KeyQueryFilter} */ + key => key.toString() !== shardKey.toString(), + /** @type {KeyQueryFilter} */ + key => key.toString() !== shardReadmeKey.toString() + ] + } + + const { prefix } = q + if (prefix != null) { + tq.filters.push((key) => { + return this._invertKey(key).toString().startsWith(prefix) + }) + } + + if (q.filters != null) { + const filters = q.filters.map(f => { + /** @type {KeyQueryFilter} */ + const filter = (key) => { + return f(this._invertKey(key)) + } + + return filter + }) + tq.filters = tq.filters.concat(filters) + } + + if (q.orders != null) { + tq.orders = q.orders.map(o => { + /** @type {KeyQueryOrder} */ + const order = (a, b) => o(this._invertKey(a), this._invertKey(b)) + + return order + }) + } + + return this.child.queryKeys(tq, options) + } + close () { return this.child.close() } diff --git a/test/mount.spec.js b/test/mount.spec.js index 9dad023..1984cb2 100644 --- a/test/mount.spec.js +++ b/test/mount.spec.js @@ -4,14 +4,15 @@ const { expect, assert } = require('aegir/utils/chai') const all = require('it-all') -const { Key, MemoryDatastore, utils: { utf8Encoder } } = require('interface-datastore') +const { Key, MemoryDatastore } = require('interface-datastore') const MountStore = require('../src').MountDatastore +const uint8ArrayFromString = require('uint8arrays/from-string') describe('MountStore', () => { it('put - no mount', async () => { const m = new MountStore([]) try { - await m.put(new Key('hello'), utf8Encoder.encode('foo')) + await m.put(new Key('hello'), uint8ArrayFromString('foo')) assert(false, 'Failed to throw error on no mount') } catch (err) { expect(err).to.be.an('Error') @@ -24,7 +25,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) try { - await m.put(new Key('/fail/hello'), utf8Encoder.encode('foo')) + await m.put(new Key('/fail/hello'), uint8ArrayFromString('foo')) assert(false, 'Failed to throw error on wrong mount') } catch (err) { expect(err).to.be.an('Error') @@ -38,7 +39,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = utf8Encoder.encode('hello') + const val = uint8ArrayFromString('hello') await m.put(new Key('/cool/hello'), val) const res = await mds.get(new Key('/hello')) expect(res).to.eql(val) @@ -51,7 +52,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = utf8Encoder.encode('hello') + const val = uint8ArrayFromString('hello') await mds.put(new Key('/hello'), val) const res = await m.get(new Key('/cool/hello')) expect(res).to.eql(val) @@ -64,7 +65,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = utf8Encoder.encode('hello') + const val = uint8ArrayFromString('hello') await mds.put(new Key('/hello'), val) const exists = await m.has(new Key('/cool/hello')) expect(exists).to.eql(true) @@ -77,7 +78,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = utf8Encoder.encode('hello') + const val = uint8ArrayFromString('hello') await m.put(new Key('/cool/hello'), val) await m.delete(new Key('/cool/hello')) let exists = await m.has(new Key('/cool/hello')) @@ -93,7 +94,7 @@ describe('MountStore', () => { prefix: new Key('cool') }]) - const val = utf8Encoder.encode('hello') + const val = uint8ArrayFromString('hello') await m.put(new Key('/cool/hello'), val) const res = await all(m.query({ prefix: '/cool' })) expect(res).to.eql([{ key: new Key('/cool/hello'), value: val }]) diff --git a/test/namespace.spec.js b/test/namespace.spec.js index 2f29cec..1b220c3 100644 --- a/test/namespace.spec.js +++ b/test/namespace.spec.js @@ -2,9 +2,10 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const { Key, MemoryDatastore, utils: { utf8Encoder } } = require('interface-datastore') +const { Key, MemoryDatastore } = require('interface-datastore') const NamespaceStore = require('../src/').NamespaceDatastore const all = require('it-all') +const uint8ArrayFromString = require('uint8arrays/from-string') describe('KeyTransformDatastore', () => { const prefixes = [ @@ -24,7 +25,7 @@ describe('KeyTransformDatastore', () => { 'foo/bar/baz/barb' ].map((s) => new Key(s)) - await Promise.all(keys.map(key => store.put(key, utf8Encoder.encode(key.toString())))) + await Promise.all(keys.map(key => store.put(key, uint8ArrayFromString(key.toString())))) const nResults = Promise.all(keys.map((key) => store.get(key))) const mResults = Promise.all(keys.map((key) => mStore.get(new Key(prefix).child(key)))) const results = await Promise.all([nResults, mResults]) diff --git a/test/sharding.spec.js b/test/sharding.spec.js index f037bc1..e6dd687 100644 --- a/test/sharding.spec.js +++ b/test/sharding.spec.js @@ -2,7 +2,9 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const { Key, MemoryDatastore, utils: { utf8Decoder, utf8Encoder } } = require('interface-datastore') +const { Key, MemoryDatastore } = require('interface-datastore') +const uint8ArrayFromString = require('uint8arrays/from-string') +const uint8ArrayToString = require('uint8arrays/to-string') const ShardingStore = require('../src').ShardingDatastore const sh = require('../src').shard @@ -17,8 +19,8 @@ describe('ShardingStore', () => { ms.get(new Key(sh.SHARDING_FN)), ms.get(new Key(sh.README_FN)) ]) - expect(utf8Decoder.decode(res[0])).to.eql(shard.toString() + '\n') - expect(utf8Decoder.decode(res[1])).to.eql(sh.readme) + expect(uint8ArrayToString(res[0])).to.eql(shard.toString() + '\n') + expect(uint8ArrayToString(res[1])).to.eql(sh.readme) }) it('open - empty', () => { @@ -43,9 +45,9 @@ describe('ShardingStore', () => { const shard = new sh.NextToLast(2) const store = new ShardingStore(ms, shard) await store.open() - await store.put(new Key('hello'), utf8Encoder.encode('test')) + await store.put(new Key('hello'), uint8ArrayFromString('test')) const res = await ms.get(new Key('ll').child(new Key('hello'))) - expect(res).to.eql(utf8Encoder.encode('test')) + expect(res).to.eql(uint8ArrayFromString('test')) }) describe('interface-datastore', () => { diff --git a/test/tiered.spec.js b/test/tiered.spec.js index d7d7244..3aa5d1a 100644 --- a/test/tiered.spec.js +++ b/test/tiered.spec.js @@ -2,8 +2,9 @@ 'use strict' const { expect } = require('aegir/utils/chai') -const { Key, MemoryDatastore, utils: { utf8Encoder } } = require('interface-datastore') +const { Key, MemoryDatastore } = require('interface-datastore') const { TieredDatastore } = require('../src') +const uint8ArrayFromString = require('uint8arrays/from-string') /** * @typedef {import('interface-datastore/dist/src/types').Datastore} Datastore */ @@ -22,7 +23,7 @@ describe('Tiered', () => { it('put', async () => { const k = new Key('hello') - const v = utf8Encoder.encode('world') + const v = uint8ArrayFromString('world') await store.put(k, v) const res = await Promise.all([ms[0].get(k), ms[1].get(k)]) res.forEach((val) => { @@ -32,7 +33,7 @@ describe('Tiered', () => { it('get and has, where available', async () => { const k = new Key('hello') - const v = utf8Encoder.encode('world') + const v = uint8ArrayFromString('world') await ms[1].put(k, v) const val = await store.get(k) expect(val).to.be.eql(v) @@ -46,7 +47,7 @@ describe('Tiered', () => { it('has and delete', async () => { const k = new Key('hello') - const v = utf8Encoder.encode('world') + const v = uint8ArrayFromString('world') await store.put(k, v) let res = await Promise.all([ms[0].has(k), ms[1].has(k)]) expect(res).to.be.eql([true, true]) diff --git a/tsconfig.json b/tsconfig.json index e605b61..fe3cbd6 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,10 +1,10 @@ { - "extends": "./node_modules/aegir/src/config/tsconfig.aegir.json", + "extends": "aegir/src/config/tsconfig.aegir.json", "compilerOptions": { "outDir": "dist" }, "include": [ - "test", // remove this line if you don't want to type-check tests + "test", "src" ] }