diff --git a/package-lock.json b/package-lock.json index 66bf3b1..b42e969 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,14 +6,14 @@ "packages": { "": { "name": "@web3-storage/gateway-lib", - "version": "2.0.3", + "version": "3.0.0", "license": "Apache-2.0 OR MIT", "dependencies": { "@ipld/car": "^5.1.0", "@web3-storage/handlebars": "^1.0.0", "bytes": "^3.1.2", "chardet": "^1.5.0", - "dagula": "^6.0.2", + "dagula": "^7.0.0", "magic-bytes.js": "^1.0.12", "mrmime": "^1.0.1", "multiformats": "^11.0.1", @@ -127,6 +127,144 @@ "npm": ">=7.0.0" } }, + "node_modules/@chainsafe/libp2p-yamux": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@chainsafe/libp2p-yamux/-/libp2p-yamux-4.0.2.tgz", + "integrity": "sha512-p0m/4ab4JLaIQqUtxvm8bSqdt9sb0uXX8PFj1CQM1eJLeV1LxzzygaSOeLxN/5ckHCuK7q/9eb9xybvl6vz/JA==", + "dependencies": { + "@libp2p/interface-connection": "^5.1.0", + "@libp2p/interface-stream-muxer": "^4.1.2", + "@libp2p/interfaces": "^3.3.2", + "@libp2p/logger": "^2.0.7", + "abortable-iterator": "^5.0.1", + "any-signal": "^4.1.1", + "it-pipe": "^3.0.1", + "it-pushable": "^3.1.3", + "uint8arraylist": "^2.4.3" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/@libp2p/interface-connection": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface-connection/-/interface-connection-5.1.0.tgz", + "integrity": "sha512-KFjCnGvFVlu0hHS/O8NOsst32mIzUQEkRWq5EhOBehXjjpOJBcm8XQaqmhBlxVfHEYm7XQsztEtFumveszzm1A==", + "dependencies": { + "@libp2p/interface-peer-id": "^2.0.0", + "@libp2p/interfaces": "^3.0.0", + "@multiformats/multiaddr": "^12.0.0", + "it-stream-types": "^2.0.1", + "uint8arraylist": "^2.4.3" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/@libp2p/interface-stream-muxer": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/@libp2p/interface-stream-muxer/-/interface-stream-muxer-4.1.2.tgz", + "integrity": "sha512-dQJcn67UaAa8YQFRJDhbo4uT453z/2lCzD/ZwTk1YOqJxATXbXgVcB8dXDQFEUiUX3ZjVQ1IBu+NlQd+IZ++zw==", + "dependencies": { + "@libp2p/interface-connection": "^5.0.0", + "@libp2p/interfaces": "^3.0.0", + "@libp2p/logger": "^2.0.7", + "abortable-iterator": "^5.0.1", + "any-signal": "^4.1.1", + "it-pushable": "^3.1.3", + "it-stream-types": "^2.0.1", + "uint8arraylist": "^2.4.3" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/@multiformats/multiaddr": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-12.1.3.tgz", + "integrity": "sha512-rNcS3njkkSwuGF4x58L47jGH5kBXBfJPNsWnrt0gujhNYn6ReDt1je7vEU5/ddrVj0TStgxw+Hm+TkYDK0b60w==", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1", + "@chainsafe/netmask": "^2.0.0", + "@libp2p/interfaces": "^3.3.1", + "dns-over-http-resolver": "^2.1.0", + "multiformats": "^11.0.0", + "uint8arrays": "^4.0.2", + "varint": "^6.0.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/abortable-iterator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/abortable-iterator/-/abortable-iterator-5.0.1.tgz", + "integrity": "sha512-hlZ5Z8UwqrKsJcelVPEqDduZowJPBQJ9ZhBC2FXpja3lXy8X6MoI5uMzIgmrA8+3jcVnp8TF/tx+IBBqYJNUrg==", + "dependencies": { + "get-iterator": "^2.0.0", + "it-stream-types": "^2.0.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/any-signal": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/any-signal/-/any-signal-4.1.1.tgz", + "integrity": "sha512-iADenERppdC+A2YKbOXXB2WUeABLaM6qnpZ70kZbPZ1cZMMJ7eF+3CaYm+/PhBizgkzlvssC7QuHS30oOiQYWA==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/it-merge": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/it-merge/-/it-merge-3.0.1.tgz", + "integrity": "sha512-I6hjU1ABO+k3xY1H6JtCSDXvUME88pxIXSgKeT4WI5rPYbQzpr98ldacVuG95WbjaJxKl6Qot6lUdxduLBQPHA==", + "dependencies": { + "it-pushable": "^3.1.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/it-pipe": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/it-pipe/-/it-pipe-3.0.1.tgz", + "integrity": "sha512-sIoNrQl1qSRg2seYSBH/3QxWhJFn9PKYvOf/bHdtCBF0bnghey44VyASsWzn5dAx0DCDDABq1hZIuzKmtBZmKA==", + "dependencies": { + "it-merge": "^3.0.0", + "it-pushable": "^3.1.2", + "it-stream-types": "^2.0.1" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/libp2p-yamux/node_modules/it-stream-types": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/it-stream-types/-/it-stream-types-2.0.1.tgz", + "integrity": "sha512-6DmOs5r7ERDbvS4q8yLKENcj6Yecr7QQTqWApbZdfAUTEC947d+PEha7PCqhm//9oxaLYL7TWRekwhoXl2s6fg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@chainsafe/netmask": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@chainsafe/netmask/-/netmask-2.0.0.tgz", + "integrity": "sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.2.0.tgz", @@ -619,18 +757,18 @@ } }, "node_modules/@libp2p/interfaces": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/@libp2p/interfaces/-/interfaces-3.3.1.tgz", - "integrity": "sha512-3N+goQt74SmaVOjwpwMPKLNgh1uDQGw8GD12c40Kc86WOq0qvpm3NfACW+H8Su2X6KmWjCSMzk9JWs9+8FtUfg==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/@libp2p/interfaces/-/interfaces-3.3.2.tgz", + "integrity": "sha512-p/M7plbrxLzuQchvNwww1Was7ZeGE2NaOFulMaZBYIihU8z3fhaV+a033OqnC/0NTX/yhfdNOG7znhYq3XoR/g==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" } }, "node_modules/@libp2p/logger": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@libp2p/logger/-/logger-2.0.6.tgz", - "integrity": "sha512-PfTGCBT6buiGeww7heG1JucBK2io2sJ2hntNh+gTVohRy4FyEvZixnWfIVD2rCM8EsbZu3Hmt/qqetzX5BrziQ==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@libp2p/logger/-/logger-2.0.7.tgz", + "integrity": "sha512-Zp9C9lMNGfVFTMVc7NvxuxMvIE6gyxDapQc/TqZH02IuIDl1JpZyCgNILr0APd8wcUxwvwRXYNf3kQ0Lmz7tuQ==", "dependencies": { "@libp2p/interface-peer-id": "^2.0.0", "debug": "^4.3.3", @@ -1678,11 +1816,12 @@ } }, "node_modules/dagula": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/dagula/-/dagula-6.0.2.tgz", - "integrity": "sha512-NzIwcSMvfIJhDsFDFInPuSPZDwHvFNNq18A+NCzICUNYd0iEpA9+BGAj+WF337g6pqqf54IogOrW99XhxbJjzQ==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/dagula/-/dagula-7.0.0.tgz", + "integrity": "sha512-DqZC/SMWId/3M7E524fuKM3UUm/CJjfuXLRbYtS0RfMOZtzzbGaISq43aRlzfdt7JvnZa4VbC6ZoPj5JDsPZWQ==", "dependencies": { "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^4.0.2", "@ipld/car": "^5.0.3", "@ipld/dag-cbor": "^9.0.0", "@ipld/dag-json": "^10.0.0", @@ -3582,9 +3721,9 @@ } }, "node_modules/it-pushable": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/it-pushable/-/it-pushable-3.1.2.tgz", - "integrity": "sha512-zU9FbeoGT0f+yobwm8agol2OTMXbq4ZSWLEi7hug6TEZx4qVhGhGyp31cayH04aBYsIoO2Nr5kgMjH/oWj2BJQ==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/it-pushable/-/it-pushable-3.1.3.tgz", + "integrity": "sha512-f50iQ85HISS6DaWCyrqf9QJ6G/kQtKIMf9xZkgZgyOvxEQDfn8OfYcLXXquCqgoLboxQtAW1ZFZyFIAsLHDtJw==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" diff --git a/package.json b/package.json index fddf28d..00d128c 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "@web3-storage/handlebars": "^1.0.0", "bytes": "^3.1.2", "chardet": "^1.5.0", - "dagula": "^6.0.2", + "dagula": "^7.0.0", "magic-bytes.js": "^1.0.12", "mrmime": "^1.0.1", "multiformats": "^11.0.1", diff --git a/src/handlers/car.js b/src/handlers/car.js index d2babd7..9bcbb26 100644 --- a/src/handlers/car.js +++ b/src/handlers/car.js @@ -6,8 +6,12 @@ import { HttpError } from '../util/errors.js' /** * @typedef {import('../bindings').IpfsUrlContext & import('../bindings').DagulaContext & { timeoutController?: import('../bindings').TimeoutControllerContext['timeoutController'] }} CarHandlerContext * @typedef {import('multiformats').CID} CID + * @typedef {{ version: 1|2, order: import('dagula').BlockOrder, dups: boolean }} CarParams */ +/** @type {CarParams} */ +const DefaultCarParams = { version: 1, order: 'unk', dups: true } + /** @type {import('../bindings').Handler} */ export async function handleCar (request, env, ctx) { const { dataCid, path, timeoutController: controller, dagula, searchParams } = ctx @@ -15,7 +19,8 @@ export async function handleCar (request, env, ctx) { if (path == null) throw new Error('missing URL path') if (!dagula) throw new Error('missing dagula instance') - const carScope = getCarScope(searchParams) + const dagScope = getDagScope(searchParams) + const { version, order, dups } = getAcceptParams(request.headers) // Use root CID for etag even tho we may resolve a different root for the terminus of the path // as etags are only relevant per path. If the caller has an etag for this path already, and @@ -34,7 +39,8 @@ export async function handleCar (request, env, ctx) { const { writer, out } = CarWriter.create(dataCid) ;(async () => { try { - for await (const block of dagula.getPath(`${dataCid}${path}`, { carScope, signal: controller?.signal })) { + for await (const block of dagula.getPath(`${dataCid}${path}`, { dagScope, order, signal: controller?.signal })) { + // @ts-expect-error await writer.put(block) } } catch (/** @type {any} */ err) { @@ -52,7 +58,7 @@ export async function handleCar (request, env, ctx) { const headers = { // Make it clear we don't support range-requests over a car stream 'Accept-Ranges': 'none', - 'Content-Type': 'application/vnd.ipld.car; version=1', + 'Content-Type': `application/vnd.ipld.car; version=${version}; order=${order}; dups=${dups ? 'y' : 'n'}`, 'X-Content-Type-Options': 'nosniff', Etag: etag, 'Cache-Control': 'public, max-age=29030400, immutable', @@ -62,11 +68,45 @@ export async function handleCar (request, env, ctx) { return new Response(toReadableStream(out), { headers }) } -/** @param {URLSearchParams} searchParams */ -function getCarScope (searchParams) { - const carScope = searchParams.get('car-scope') ?? 'all' - if (carScope === 'all' || carScope === 'file' || carScope === 'block') { - return carScope +/** + * @param {URLSearchParams} searchParams + * @returns {import('dagula').DagScope} + */ +function getDagScope (searchParams) { + const scope = searchParams.get('dag-scope') ?? 'all' + if (scope === 'all' || scope === 'entity' || scope === 'block') { + return scope + } + throw new HttpError(`unsupported dag-scope: ${scope}`, { status: 400 }) +} + +/** + * @param {Headers} headers + * @returns {CarParams} + */ +function getAcceptParams (headers) { + const accept = headers.get('accept') + if (!accept) return DefaultCarParams + + const types = accept.split(',').map(s => s.trim()) + const carType = types.find(t => t.startsWith('application/vnd.ipld.car')) + if (!carType) return DefaultCarParams + + const paramPairs = carType.split(';').slice(1).map(s => s.trim()) + const { version, order, dups } = Object.fromEntries(paramPairs.map(p => p.split('=').map(s => s.trim()))) + + // only CARv1 + if (version != null && version !== '1') { + throw new HttpError(`unsupported accept parameter: version=${version}`, { status: 400 }) } - throw new HttpError(`unsupported car-scope: ${carScope}`, { status: 400 }) + // only yes duplicates + if (dups && dups !== 'y') { + throw new HttpError(`unsupported accept parameter: dups=${dups}`, { status: 400 }) + } + // only dfs or unk ordering + if (order && order !== 'dfs' && order !== 'unk') { + throw new HttpError(`unsupported accept parameter: order=${order}`, { status: 400 }) + } + + return { version: 1, order, dups: true } } diff --git a/test/handlers/car.spec.js b/test/handlers/car.spec.js index 9ca4a91..c42ffb2 100644 --- a/test/handlers/car.spec.js +++ b/test/handlers/car.spec.js @@ -30,4 +30,104 @@ describe('CAR handler', () => { assert(await reader.get(leafBlock.cid)) assert(await reader.get(rootBlock.cid)) }) + + it('serves a CAR with query parameter dag-scope=block', async () => { + const waitUntil = mockWaitUntil() + const leafBlock0 = await encode({ value: fromString('test0'), codec: raw, hasher }) + const leafBlock1 = await encode({ value: fromString('test1'), codec: raw, hasher }) + const rootBlock = await encode({ value: { leaf0: leafBlock0.cid, leaf1: leafBlock1.cid }, codec: cbor, hasher }) + const blockstore = mockBlockstore([leafBlock0, leafBlock1, rootBlock]) + const dagula = new Dagula(blockstore) + const url = new URL(`http://localhost/ipfs/${rootBlock.cid}/leaf0?dag-scope=block`) + const path = url.pathname.replace(`/ipfs/${rootBlock.cid}`, '') + const ctx = { waitUntil, dagula, dataCid: rootBlock.cid, path, searchParams: url.searchParams } + const env = { DEBUG: 'true' } + const req = new Request(url) + const res = await handleCar(req, env, ctx) + const reader = await CarReader.fromBytes(new Uint8Array(await res.arrayBuffer())) + const roots = await reader.getRoots() + assert.strictEqual(roots[0].toString(), rootBlock.cid.toString()) + assert(await reader.get(leafBlock0.cid)) + assert(await reader.get(rootBlock.cid)) + assert(!(await reader.has(leafBlock1.cid))) + }) + + it('serves a CAR with accept parameter order=dfs', async () => { + const waitUntil = mockWaitUntil() + + const leafBlock0 = await encode({ value: fromString('test0'), codec: raw, hasher }) + const leafBlock1 = await encode({ value: fromString('test1'), codec: raw, hasher }) + const branchBlock0 = await encode({ value: { leaf: leafBlock0.cid }, codec: cbor, hasher }) + const branchBlock1 = await encode({ value: { leaf: leafBlock1.cid }, codec: cbor, hasher }) + const rootBlock = await encode({ value: [branchBlock0.cid, branchBlock1.cid], codec: cbor, hasher }) + const blockstore = mockBlockstore([leafBlock0, leafBlock1, branchBlock0, branchBlock1, rootBlock]) + const dagula = new Dagula(blockstore) + const url = new URL(`http://localhost/ipfs/${rootBlock.cid}`) + const path = url.pathname.replace(`/ipfs/${rootBlock.cid}`, '') + const ctx = { waitUntil, dagula, dataCid: rootBlock.cid, path, searchParams: url.searchParams } + const env = { DEBUG: 'true' } + const req = new Request(url, { + headers: { + Accept: 'application/vnd.ipld.car; order=dfs' + } + }) + const res = await handleCar(req, env, ctx) + const contentType = res.headers.get('Content-Type') + assert(contentType) + assert(contentType.includes('order=dfs')) + + const reader = await CarReader.fromBytes(new Uint8Array(await res.arrayBuffer())) + const roots = await reader.getRoots() + assert.strictEqual(roots[0].toString(), rootBlock.cid.toString()) + + const blocks = [] + for await (const b of reader.blocks()) { + blocks.push(b) + } + assert.strictEqual(blocks[0].cid.toString(), rootBlock.cid.toString()) + assert.strictEqual(blocks[1].cid.toString(), branchBlock0.cid.toString()) + assert.strictEqual(blocks[2].cid.toString(), leafBlock0.cid.toString()) + assert.strictEqual(blocks[3].cid.toString(), branchBlock1.cid.toString()) + assert.strictEqual(blocks[4].cid.toString(), leafBlock1.cid.toString()) + }) + + // unk = unknown, so underlying implementation could change in the future + it('serves a CAR with accept parameter order=unk', async () => { + const waitUntil = mockWaitUntil() + + const leafBlock0 = await encode({ value: fromString('test0'), codec: raw, hasher }) + const leafBlock1 = await encode({ value: fromString('test1'), codec: raw, hasher }) + const branchBlock0 = await encode({ value: { leaf: leafBlock0.cid }, codec: cbor, hasher }) + const branchBlock1 = await encode({ value: { leaf: leafBlock1.cid }, codec: cbor, hasher }) + const rootBlock = await encode({ value: [branchBlock0.cid, branchBlock1.cid], codec: cbor, hasher }) + const blockstore = mockBlockstore([leafBlock0, leafBlock1, branchBlock0, branchBlock1, rootBlock]) + const dagula = new Dagula(blockstore) + const url = new URL(`http://localhost/ipfs/${rootBlock.cid}`) + const path = url.pathname.replace(`/ipfs/${rootBlock.cid}`, '') + const ctx = { waitUntil, dagula, dataCid: rootBlock.cid, path, searchParams: url.searchParams } + const env = { DEBUG: 'true' } + const req = new Request(url, { + headers: { + Accept: 'application/vnd.ipld.car; order=unk' + } + }) + const res = await handleCar(req, env, ctx) + const contentType = res.headers.get('Content-Type') + assert(contentType) + assert(contentType.includes('order=unk')) + + const reader = await CarReader.fromBytes(new Uint8Array(await res.arrayBuffer())) + const roots = await reader.getRoots() + assert.strictEqual(roots[0].toString(), rootBlock.cid.toString()) + + const blocks = [] + for await (const b of reader.blocks()) { + blocks.push(b) + } + assert.strictEqual(blocks[0].cid.toString(), rootBlock.cid.toString()) + assert.strictEqual(blocks[1].cid.toString(), branchBlock0.cid.toString()) + assert.strictEqual(blocks[2].cid.toString(), branchBlock1.cid.toString()) + assert.strictEqual(blocks[3].cid.toString(), leafBlock0.cid.toString()) + assert.strictEqual(blocks[4].cid.toString(), leafBlock1.cid.toString()) + }) })