From 1c7b7a6faff866bb7c8fece427777e33e7e27416 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 17 May 2024 12:44:30 +0100 Subject: [PATCH] feat: fetch blocks from location URLs (#103) Freeway now attempts to `fetch` data from claimed location URLs and does not rely on an R2 bucket. --- package-lock.json | 364 +++++++++++++-------------------- package.json | 3 +- src/lib/blockstore.js | 29 ++- src/lib/dag-index/entry.js | 32 +-- test/helpers/bucket.js | 74 +++++++ test/helpers/content-claims.js | 8 +- test/index.spec.js | 60 +++--- 7 files changed, 281 insertions(+), 289 deletions(-) create mode 100644 test/helpers/bucket.js diff --git a/package-lock.json b/package-lock.json index 42992a9..4e2f961 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,7 +14,7 @@ "@ipld/dag-json": "^10.1.7", "@ipld/dag-pb": "^4.0.8", "@web3-storage/content-claims": "^4.0.5", - "@web3-storage/gateway-lib": "^5.0.0", + "@web3-storage/gateway-lib": "^5.0.1", "cardex": "^3.0.0", "dagula": "^7.3.0", "http-range-parse": "^1.0.0", @@ -26,6 +26,7 @@ "@aws-sdk/client-s3": "^3.490.0", "@cloudflare/workers-types": "^4.20231218.0", "@ucanto/principal": "^8.1.0", + "@web3-storage/public-r2-bucket": "^1.2.1", "ava": "^5.3.1", "byteranges": "^1.1.0", "carbites": "^1.0.6", @@ -34,7 +35,7 @@ "esbuild": "^0.18.20", "files-from-path": "^0.2.6", "ipfs-car": "^0.9.2", - "miniflare": "^2.14.1", + "miniflare": "^2.14.2", "standard": "^17.1.0", "typescript": "^5.3.3", "uint8arrays": "^4.0.10" @@ -3710,39 +3711,27 @@ } }, "node_modules/@miniflare/cache": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/cache/-/cache-2.14.1.tgz", - "integrity": "sha512-f/o6UBV6UX+MlhjcEch73/wjQvvNo37dgYmP6Pn2ax1/mEHhJ7allNAqenmonT4djNeyB3eEYV3zUl54wCEwrg==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/cache/-/cache-2.14.2.tgz", + "integrity": "sha512-XH218Y2jxSOfxG8EyuprBKhI/Fn6xLrb9A39niJBlzpiKXqr8skl/sy/sUL5tfvqEbEnqDagGne8zEcjM+1fBg==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", "http-cache-semantics": "^4.1.0", - "undici": "5.20.0" + "undici": "5.28.2" }, "engines": { "node": ">=16.13" } }, - "node_modules/@miniflare/cache/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miniflare/cli-parser": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/cli-parser/-/cli-parser-2.14.1.tgz", - "integrity": "sha512-MLvMuvxQPL/uw94Dg6CR3CM62uGoxxCkgF7aDtBYF9zb6YCfjOQn0q5d7zWxydpYAb5Dn1N1hZMnjAy7mlpNdQ==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/cli-parser/-/cli-parser-2.14.2.tgz", + "integrity": "sha512-CzC7OnPWuMWSJrmnn0PUToMFDkMEnsFFE+ybA+Gqpgcdb/gaLz+yP0/Hagb5YN4JMxh5SBG7NCktsjZKaO94ig==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1", + "@miniflare/shared": "2.14.2", "kleur": "^4.1.4" }, "engines": { @@ -3750,20 +3739,20 @@ } }, "node_modules/@miniflare/core": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/core/-/core-2.14.1.tgz", - "integrity": "sha512-d+SGAda/VoXq+SKz04oq8ATUwQw5755L87fgPR8pTdR2YbWkxdbmEm1z2olOpDiUjcR86aN6NtCjY6tUC7fqaw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/core/-/core-2.14.2.tgz", + "integrity": "sha512-n/smm5ZTg7ilGM4fxO7Gxhbe573oc8Za06M3b2fO+lPWqF6NJcEKdCC+sJntVFbn3Cbbd2G1ChISmugPfmlCkQ==", "dev": true, "dependencies": { "@iarna/toml": "^2.2.5", - "@miniflare/queues": "2.14.1", - "@miniflare/shared": "2.14.1", - "@miniflare/watcher": "2.14.1", + "@miniflare/queues": "2.14.2", + "@miniflare/shared": "2.14.2", + "@miniflare/watcher": "2.14.2", "busboy": "^1.6.0", "dotenv": "^10.0.0", "kleur": "^4.1.4", "set-cookie-parser": "^2.4.8", - "undici": "5.20.0", + "undici": "5.28.2", "urlpattern-polyfill": "^4.0.3" }, "engines": { @@ -3779,97 +3768,61 @@ "node": ">=10" } }, - "node_modules/@miniflare/core/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miniflare/d1": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/d1/-/d1-2.14.1.tgz", - "integrity": "sha512-MulDDBsDD8o5DwiqdMeJZy2vLoMji+NWnLcuibSag2mayA0LJcp0eHezseZNkW+knciWR1gMP8Xpa4Q1KwkbKA==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/d1/-/d1-2.14.2.tgz", + "integrity": "sha512-3NPJyBLbFfzz9VAAdIZrDRdRpyslVCJoZHQk0/0CX3z2mJIfcQzjZhox2cYCFNH8NMJ7pRg6AeSMPYAnDKECDg==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1" + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2" }, "engines": { "node": ">=16.7" } }, "node_modules/@miniflare/durable-objects": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/durable-objects/-/durable-objects-2.14.1.tgz", - "integrity": "sha512-T+oHGw5GcEIilkzrf0xDES7jzLVqcXJzSGsEIWqnBFLtdlKmrZF679ulRLBbyMVgvpQz6FRONh9jTH1XIiuObQ==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/durable-objects/-/durable-objects-2.14.2.tgz", + "integrity": "sha512-BfK+ZkJABoi7gd/O6WbpsO4GrgW+0dmOBWJDlNBxQ7GIpa+w3n9+SNnrYUxKzWlPSvz+TfTTk381B1z/Z87lPw==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", - "@miniflare/storage-memory": "2.14.1", - "undici": "5.20.0" + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", + "@miniflare/storage-memory": "2.14.2", + "undici": "5.28.2" }, "engines": { "node": ">=16.13" } }, - "node_modules/@miniflare/durable-objects/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miniflare/html-rewriter": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/html-rewriter/-/html-rewriter-2.14.1.tgz", - "integrity": "sha512-vp4uZXuEKhtIaxoXa7jgDAPItlzjbfoUqYWp+fwDKv4J4mfQnzzs/5hwjbE7+Ihm/KNI0zNi8P0sSWjIRFl6ng==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/html-rewriter/-/html-rewriter-2.14.2.tgz", + "integrity": "sha512-tu0kd9bj38uZ04loHb3sMI8kzUzZPgPOAJEdS9zmdSPh0uOkjCDf/TEkKsDdv2OFysyb0DRsIrwhPqCTIrPf1Q==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", "html-rewriter-wasm": "^0.4.1", - "undici": "5.20.0" + "undici": "5.28.2" }, "engines": { "node": ">=16.13" } }, - "node_modules/@miniflare/html-rewriter/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miniflare/http-server": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/http-server/-/http-server-2.14.1.tgz", - "integrity": "sha512-ICT9KuecAF5+AHs9dTnk7DRxCPYLfI3QCtfHOcGQWFuOoe4MCtQk4joo0pxMaSgAJ4TiOOxz3KRWO0G2FwgiPQ==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/http-server/-/http-server-2.14.2.tgz", + "integrity": "sha512-cc8OfZahdPd7pDER3xR1Io29g4pLrVhhxYnoT7t2TbhLoxOl93tRjxdUPX/UEjmy0MCYS4mutpSoWx49FB9OcA==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", - "@miniflare/web-sockets": "2.14.1", + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", + "@miniflare/web-sockets": "2.14.2", "kleur": "^4.1.4", "selfsigned": "^2.0.0", - "undici": "5.20.0", + "undici": "5.28.2", "ws": "^8.2.2", "youch": "^2.2.2" }, @@ -3877,88 +3830,64 @@ "node": ">=16.13" } }, - "node_modules/@miniflare/http-server/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miniflare/kv": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/kv/-/kv-2.14.1.tgz", - "integrity": "sha512-Gp07Wcszle7ptsoO8mCtKQRs0AbQnYo1rgnxUcsTL3xJJaHXEA/B9EKSADS2XzJMeY4PgUOHU6Rf08OOF2yWag==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/kv/-/kv-2.14.2.tgz", + "integrity": "sha512-3rs4cJOGACT/U7NH7j4KD29ugXRYUiM0aGkvOEdFQtChXLsYClJljXpezTfJJxBwZjdS4F2UFTixtFcHp74UfA==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1" + "@miniflare/shared": "2.14.2" }, "engines": { "node": ">=16.13" } }, "node_modules/@miniflare/queues": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/queues/-/queues-2.14.1.tgz", - "integrity": "sha512-uBzrbBkIgtNoztDpmMMISg/brYtxLHRE7oTaN8OVnq3bG+3nF9kQC42HUz+Vg+sf65UlvhSaqkjllgx+fNtOxQ==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/queues/-/queues-2.14.2.tgz", + "integrity": "sha512-OylkRs4lOWKvGnX+Azab3nx+1qwC87M36/hkgAU1RRvVDCOxOrYLvNLUczFfgmgMBwpYsmmW8YOIASlI3p4Qgw==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1" + "@miniflare/shared": "2.14.2" }, "engines": { "node": ">=16.7" } }, "node_modules/@miniflare/r2": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/r2/-/r2-2.14.1.tgz", - "integrity": "sha512-grOMnGf2XSicbgxMYMBfWE37k/e7l5NnwXZIViQ+N06uksp+MLA8E6yKQNtvrWQS66TM8gBvMnWo96OFmYjb6Q==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/r2/-/r2-2.14.2.tgz", + "integrity": "sha512-uuc7dx6OqSQT8i0F2rsigfizXmInssRvvJAjoi1ltaNZNJCHH9l1PwHfaNc/XAuDjYmiCjtHDaPdRvZU9g9F3g==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", - "undici": "5.20.0" + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", + "undici": "5.28.2" }, "engines": { "node": ">=16.13" } }, - "node_modules/@miniflare/r2/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miniflare/runner-vm": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/runner-vm/-/runner-vm-2.14.1.tgz", - "integrity": "sha512-UobsGM0ICVPDlJD54VPDSx0EXrIY3nJMXBy2zIFuuUOz4hQKXvMQ6jtAlJ8UNKer+XXI3Mb/9R/gfU8r6kxIMA==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/runner-vm/-/runner-vm-2.14.2.tgz", + "integrity": "sha512-WlyxAQ+bv/9Pm/xnbpgAg7RNX4pz/q3flytUoo4z4OrRmNEuXrbMUsJZnH8dziqzrZ2gCLkYIEzeaTmSQKp5Jg==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1" + "@miniflare/shared": "2.14.2" }, "engines": { "node": ">=16.13" } }, "node_modules/@miniflare/scheduler": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/scheduler/-/scheduler-2.14.1.tgz", - "integrity": "sha512-mCFUkGpPcV74tbDAuDLSrLPDiRei7hIIPQL53C4rtCDyLfxYgwLQFDecllQs5IvDzAx4qFpu835Ppc6tbHR5Mw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/scheduler/-/scheduler-2.14.2.tgz", + "integrity": "sha512-gJejGz9F2hZl8NHfQd2iNwDnuNsK27DkWpLHiPkIqlrbz8tglN/kUKAa0rbTOKipsdo2+h6KRqLRq5PxvJ3T8w==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", "cron-schedule": "^3.0.4" }, "engines": { @@ -3966,9 +3895,9 @@ } }, "node_modules/@miniflare/shared": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/shared/-/shared-2.14.1.tgz", - "integrity": "sha512-73GnLtWn5iP936ctE6ZJrMqGu134KOoIIveq5Yd/B+NnbFfzpuzjCpkLrnqjkDdsxDbruXSb5eTR/SmAdpJxZQ==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/shared/-/shared-2.14.2.tgz", + "integrity": "sha512-dDnYIztz10zDQjaFJ8Gy9UaaBWZkw3NyhFdpX6tAeyPA/2lGvkftc42MYmNi8s5ljqkZAtKgWAJnSf2K75NCJw==", "dev": true, "dependencies": { "@types/better-sqlite3": "^7.6.0", @@ -3981,83 +3910,71 @@ } }, "node_modules/@miniflare/sites": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/sites/-/sites-2.14.1.tgz", - "integrity": "sha512-AbbIcU6VBeaNqVgMiLMWN2a09eX3jZmjaEi0uKqufVDqW/QIz47/30aC0O9qTe+XYpi3jjph/Ux7uEY8Z+enMw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/sites/-/sites-2.14.2.tgz", + "integrity": "sha512-jFOx1G5kD+kTubsga6jcFbMdU2nSuNG2/EkojwuhYT8hYp3qd8duvPyh1V+OR2tMvM4FWu6jXPXNZNBHXHQaUQ==", "dev": true, "dependencies": { - "@miniflare/kv": "2.14.1", - "@miniflare/shared": "2.14.1", - "@miniflare/storage-file": "2.14.1" + "@miniflare/kv": "2.14.2", + "@miniflare/shared": "2.14.2", + "@miniflare/storage-file": "2.14.2" }, "engines": { "node": ">=16.13" } }, "node_modules/@miniflare/storage-file": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/storage-file/-/storage-file-2.14.1.tgz", - "integrity": "sha512-faZu9tRSW6c/looVFI/ZhkdGsIc9NfNCbSl3jJRmm7xgyZ+/S+dQ5JtGVbVsUIX8YGWDyE2j3oWCGCjxGLEpkg==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/storage-file/-/storage-file-2.14.2.tgz", + "integrity": "sha512-tn8rqMBeTtN+ICHQAMKQ0quHGYIkcyDK0qKW+Ic14gdfGDZx45BqXExQM9wTVqKtwAt85zp5eKVUYQCFfUx46Q==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1", - "@miniflare/storage-memory": "2.14.1" + "@miniflare/shared": "2.14.2", + "@miniflare/storage-memory": "2.14.2" }, "engines": { "node": ">=16.13" } }, "node_modules/@miniflare/storage-memory": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/storage-memory/-/storage-memory-2.14.1.tgz", - "integrity": "sha512-lfQbQwopVWd4W5XzrYdp0rhk3dJpvSmv1Wwn9RhNO20WrcuoxpdSzbmpBahsgYVg+OheVaEbS6RpFqdmwwLTog==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/storage-memory/-/storage-memory-2.14.2.tgz", + "integrity": "sha512-9Wtz9mayHIY0LDsfpMGx+/sfKCq3eAQJzYY+ju1tTEaKR0sVAuO51wu0wbyldsjj9OcBcd2X32iPbIa7KcSZQQ==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1" + "@miniflare/shared": "2.14.2" }, "engines": { "node": ">=16.13" } }, "node_modules/@miniflare/watcher": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/watcher/-/watcher-2.14.1.tgz", - "integrity": "sha512-dkFvetm5wk6pwunlYb/UkI0yFNb3otLpRm5RDywMUzqObEf+rCiNNAbJe3HUspr2ncZVAaRWcEaDh82vYK5cmw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/watcher/-/watcher-2.14.2.tgz", + "integrity": "sha512-/TL0np4uYDl+6MdseDApZmDdlJ6Y7AY5iDY0TvUQJG9nyBoCjX6w0Zn4SiKDwO6660rPtSqZ5c7HzbPhGb5vsA==", "dev": true, "dependencies": { - "@miniflare/shared": "2.14.1" + "@miniflare/shared": "2.14.2" }, "engines": { "node": ">=16.13" } }, "node_modules/@miniflare/web-sockets": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/@miniflare/web-sockets/-/web-sockets-2.14.1.tgz", - "integrity": "sha512-3N//L5EjF7+xXd7qCLR2ylUwm8t2MKyGPGWEtRBrQ2xqYYWhewKTjlquHCOPU5Irnnd/4BhTmFA55MNrq7m4Nw==", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/@miniflare/web-sockets/-/web-sockets-2.14.2.tgz", + "integrity": "sha512-kpbVlznPuxNQahssQvZiNPQo/iPme7qV3WMQeg6TYNCkYD7n6vEqeFZ5E/eQgB59xCanpvw4Ci8y/+SdMK6BUg==", "dev": true, "dependencies": { - "@miniflare/core": "2.14.1", - "@miniflare/shared": "2.14.1", - "undici": "5.20.0", + "@miniflare/core": "2.14.2", + "@miniflare/shared": "2.14.2", + "undici": "5.28.2", "ws": "^8.2.2" }, "engines": { "node": ">=16.13" } }, - "node_modules/@miniflare/web-sockets/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/@miyauci/isx": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@miyauci/isx/-/isx-1.1.1.tgz", @@ -5163,9 +5080,9 @@ } }, "node_modules/@types/better-sqlite3": { - "version": "7.6.8", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.8.tgz", - "integrity": "sha512-ASndM4rdGrzk7iXXqyNC4fbwt4UEjpK0i3j4q4FyeQrLAthfB6s7EF135ZJE0qQxtKIMFwmyT6x0switET7uIw==", + "version": "7.6.10", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.10.tgz", + "integrity": "sha512-TZBjD+yOsyrUJGmcUj6OS3JADk3+UZcNv3NOBqGkM09bZdi28fNZw8ODqbMOLfKCu7RYCO62/ldq1iHbzxqoPw==", "dev": true, "dependencies": { "@types/node": "*" @@ -5507,9 +5424,9 @@ } }, "node_modules/@web3-storage/gateway-lib": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@web3-storage/gateway-lib/-/gateway-lib-5.0.0.tgz", - "integrity": "sha512-jplyNJpXyv9/EUv0oQr9+35+CoHw6o8V3nzgNremUE9Akt4oyU6UKq6iutb1ARrtjVNNWr9ww8G2kVW3LgkhTA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/@web3-storage/gateway-lib/-/gateway-lib-5.0.1.tgz", + "integrity": "sha512-ofMowCarga9sxv0tgBXnbj6JXT5j0gV1L92FGjP026jR3u6t4GI4E5x0QYeNBZ4fu3j41a2zfAHuTSKWFmi9iA==", "dependencies": { "@httpland/range-parser": "^1.2.0", "@ipld/car": "^5.2.6", @@ -5593,6 +5510,17 @@ "uglify-js": "^3.1.4" } }, + "node_modules/@web3-storage/public-r2-bucket": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@web3-storage/public-r2-bucket/-/public-r2-bucket-1.2.1.tgz", + "integrity": "sha512-l5Ac3U9vxescuOBdn6j94bwZfu9CK6ZyE1fN7VWYJZyxAKqUyAGDlN8DMVmGbpvRqcfNVEdUzCuCsGZCl7zfeQ==", + "dev": true, + "dependencies": { + "@httpland/range-parser": "^1.2.0", + "@web3-storage/gateway-lib": "^5.0.1", + "multipart-byte-range": "^2.0.2" + } + }, "node_modules/@zxing/text-encoding": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/@zxing/text-encoding/-/text-encoding-0.9.0.tgz", @@ -11199,32 +11127,32 @@ } }, "node_modules/miniflare": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-2.14.1.tgz", - "integrity": "sha512-8Yqms4OZp1J1k0SGhOQ6MKWptWSXCm8Yh4kfFh11zZIlT1PoE0qXWQmAzwZnT5hFNmS3tsghu3GOgToxM+jCnA==", - "dev": true, - "dependencies": { - "@miniflare/cache": "2.14.1", - "@miniflare/cli-parser": "2.14.1", - "@miniflare/core": "2.14.1", - "@miniflare/d1": "2.14.1", - "@miniflare/durable-objects": "2.14.1", - "@miniflare/html-rewriter": "2.14.1", - "@miniflare/http-server": "2.14.1", - "@miniflare/kv": "2.14.1", - "@miniflare/queues": "2.14.1", - "@miniflare/r2": "2.14.1", - "@miniflare/runner-vm": "2.14.1", - "@miniflare/scheduler": "2.14.1", - "@miniflare/shared": "2.14.1", - "@miniflare/sites": "2.14.1", - "@miniflare/storage-file": "2.14.1", - "@miniflare/storage-memory": "2.14.1", - "@miniflare/web-sockets": "2.14.1", + "version": "2.14.2", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-2.14.2.tgz", + "integrity": "sha512-s2gruuOFT0o3cmQW+EX3k2EaLe1iHZ4OVXRmcNLj5ivrxxRHcK4Hy0LMd6bUOVDcT5KSKs7ylBkHqzp0zmUKCg==", + "dev": true, + "dependencies": { + "@miniflare/cache": "2.14.2", + "@miniflare/cli-parser": "2.14.2", + "@miniflare/core": "2.14.2", + "@miniflare/d1": "2.14.2", + "@miniflare/durable-objects": "2.14.2", + "@miniflare/html-rewriter": "2.14.2", + "@miniflare/http-server": "2.14.2", + "@miniflare/kv": "2.14.2", + "@miniflare/queues": "2.14.2", + "@miniflare/r2": "2.14.2", + "@miniflare/runner-vm": "2.14.2", + "@miniflare/scheduler": "2.14.2", + "@miniflare/shared": "2.14.2", + "@miniflare/sites": "2.14.2", + "@miniflare/storage-file": "2.14.2", + "@miniflare/storage-memory": "2.14.2", + "@miniflare/web-sockets": "2.14.2", "kleur": "^4.1.4", "semiver": "^1.1.0", "source-map-support": "^0.5.20", - "undici": "5.20.0" + "undici": "5.28.2" }, "bin": { "miniflare": "bootstrap.js" @@ -11233,7 +11161,7 @@ "node": ">=16.13" }, "peerDependencies": { - "@miniflare/storage-redis": "2.14.1", + "@miniflare/storage-redis": "2.14.2", "cron-schedule": "^3.0.4", "ioredis": "^4.27.9" }, @@ -11249,18 +11177,6 @@ } } }, - "node_modules/miniflare/node_modules/undici": { - "version": "5.20.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.20.0.tgz", - "integrity": "sha512-J3j60dYzuo6Eevbawwp1sdg16k5Tf768bxYK4TUJRH7cBM4kFCbf3mOnM/0E3vQYXvpxITbbWmBafaDbxLDz3g==", - "dev": true, - "dependencies": { - "busboy": "^1.6.0" - }, - "engines": { - "node": ">=12.18" - } - }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -11426,9 +11342,9 @@ "integrity": "sha512-bt3R5iXe2O8xpp3wkmQhC73b/lC4S2ihU8Dndwcsysqbydqb8N+bpP116qMcClZ17g58iSIwtXUTcg2zT4sniA==" }, "node_modules/multipart-byte-range": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/multipart-byte-range/-/multipart-byte-range-2.0.1.tgz", - "integrity": "sha512-WhtupMhoBKS9YzDszVBpeGqwJTBjdqTgXneqo0tJyUwlTn2h7Pbtr+mdaVV554dLU68jICgefaQfqcPkl62m3Q==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/multipart-byte-range/-/multipart-byte-range-2.0.2.tgz", + "integrity": "sha512-S1t3lY/FeYpfpVG6XD1BIRBOsGwwBXyqK/T+w9sKVNDS1OSUFxjjhTBYkRyMw2VT5oBwZInlKK58qAgB8Rcniw==", "dependencies": { "byteranges": "^1.1.0" } diff --git a/package.json b/package.json index e2af463..4edbea7 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "@ipld/dag-json": "^10.1.7", "@ipld/dag-pb": "^4.0.8", "@web3-storage/content-claims": "^4.0.5", - "@web3-storage/gateway-lib": "^5.0.0", + "@web3-storage/gateway-lib": "^5.0.1", "cardex": "^3.0.0", "dagula": "^7.3.0", "http-range-parse": "^1.0.0", @@ -53,6 +53,7 @@ "@aws-sdk/client-s3": "^3.490.0", "@cloudflare/workers-types": "^4.20231218.0", "@ucanto/principal": "^8.1.0", + "@web3-storage/public-r2-bucket": "^1.2.1", "ava": "^5.3.1", "byteranges": "^1.1.0", "carbites": "^1.0.6", diff --git a/src/lib/blockstore.js b/src/lib/blockstore.js index 0afdb9b..88bb13b 100644 --- a/src/lib/blockstore.js +++ b/src/lib/blockstore.js @@ -38,11 +38,16 @@ export class R2Blockstore { if (!entry) return if (IndexEntry.isLocated(entry)) { - const keyAndOptions = IndexEntry.toBucketGet(entry) - if (!keyAndOptions) return - - const res = await this._dataBucket.get(keyAndOptions[0], keyAndOptions[1]) - return res ? { cid, bytes: new Uint8Array(await res.arrayBuffer()) } : undefined + for (const { url, range } of IndexEntry.toRequestCandidates(entry)) { + const headers = { Range: `bytes=${range[0]}-${range[1]}` } + const res = await fetch(url, { headers }) + if (!res.ok) { + console.warn(`failed to fetch ${url}: ${res.status} ${await res.text()}`) + continue + } + return { cid, bytes: new Uint8Array(await res.arrayBuffer()) } + } + return } const carPath = `${entry.origin}/${entry.origin}.car` @@ -84,11 +89,15 @@ export class R2Blockstore { const entry = await this._idx.get(cid) if (!entry) return - const keyAndOptions = IndexEntry.toBucketGet(entry, options) - if (!keyAndOptions) return - - const res = await this._dataBucket.get(keyAndOptions[0], keyAndOptions[1]) - return /** @type {ReadableStream|undefined} */ (res?.body) + for (const { url, range } of IndexEntry.toRequestCandidates(entry, options)) { + const headers = { Range: `bytes=${range[0]}-${range[1]}` } + const res = await fetch(url, { headers }) + if (!res.ok) { + console.warn(`failed to fetch ${url}: ${res.status} ${await res.text()}`) + continue + } + return /** @type {ReadableStream|undefined} */ (res?.body) + } } } diff --git a/src/lib/dag-index/entry.js b/src/lib/dag-index/entry.js index 5b9f29a..28669a1 100644 --- a/src/lib/dag-index/entry.js +++ b/src/lib/dag-index/entry.js @@ -1,7 +1,3 @@ -import * as Link from 'multiformats/link' -import { base58btc } from 'multiformats/bases/base58' -import { CAR_CODE } from '../../constants.js' - /** * An index entry is "located" if a content claim has specified it's location * i.e. it is of type `LocatedIndexEntry`. @@ -12,28 +8,18 @@ import { CAR_CODE } from '../../constants.js' export const isLocated = entry => 'site' in entry /** + * Convert an index entry into a list of URL+byterange for requesting the + * content. + * + * @typedef {{ url: URL, range: import('dagula').AbsoluteRange }} Candidate * @param {import('./api.js').IndexEntry} entry * @param {import('dagula').RangeOptions} [options] - * @returns {[key: string, options: import('@cloudflare/workers-types').R2GetOptions]|undefined} + * @returns {Candidate[]} */ -export const toBucketGet = (entry, options) => { - if (!isLocated(entry)) return - - // if host is "w3s.link" then content can be found in CARPARK - const url = entry.site.location.find(l => l.hostname === 'w3s.link') - if (!url) return - - const link = Link.parse(url.pathname.split('/')[2]) - - let key - if (link.code === CAR_CODE) { - key = `${link}/${link}.car` - } else { - const digestString = base58btc.encode(link.multihash.bytes) - key = `${digestString}/${digestString}.blob` - } +export const toRequestCandidates = (entry, options) => { + if (!isLocated(entry)) return [] const first = entry.site.range.offset + (options?.range?.[0] ?? 0) const last = entry.site.range.offset + (options?.range?.[1] ?? (entry.site.range.length - 1)) - const range = { offset: first, length: last - first + 1 } - return [key, { range }] + const range = /** @type {import('dagula').AbsoluteRange} */ ([first, last]) + return entry.site.location.map(url => ({ url, range })) } diff --git a/test/helpers/bucket.js b/test/helpers/bucket.js new file mode 100644 index 0000000..ace09f2 --- /dev/null +++ b/test/helpers/bucket.js @@ -0,0 +1,74 @@ +import http from 'node:http' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import * as PublicBucket from '@web3-storage/public-r2-bucket' + +/** + * @typedef {{ close: () => void, getCallCount: () => number, resetCallCount: () => void, url: URL }} MockBucketService + */ + +/** @param {import('@cloudflare/workers-types').R2Bucket} bucket */ +export const mockBucketService = async (bucket) => { + let callCount = 0 + + const getCallCount = () => callCount + const resetCallCount = () => { + callCount = 0 + } + + const handler = toNodeHttpHandler(PublicBucket.handler, { BUCKET: bucket }) + + const server = http.createServer(async (request, response) => { + callCount++ + if (!['GET', 'HEAD'].includes(request.method ?? 'GET')) { + response.writeHead(405) + return response.end() + } + await handler(request, response) + }) + await new Promise(resolve => server.listen(resolve)) + const close = () => { + server.closeAllConnections() + server.close() + } + // @ts-expect-error + const { port } = server.address() + const url = new URL(`http://127.0.0.1:${port}`) + return { close, port, getCallCount, resetCallCount, url } +} + +/** + * @template E + * @template C + * @param {(request: Request, env: E, ctx?: C) => Promise} handler + * @param {E} env + * @param {C} [ctx] + */ +const toNodeHttpHandler = (handler, env, ctx) => { + /** @type {import('node:http').RequestListener} */ + return async (req, res) => { + const url = new URL(req.url || '', `http://${req.headers.host}`) + const headers = new Headers() + for (let i = 0; i < req.rawHeaders.length; i += 2) { + headers.append(req.rawHeaders[i], req.rawHeaders[i + 1]) + } + const { method } = req + const body = + /** @type {ReadableStream|undefined} */ + (['GET', 'HEAD'].includes(method ?? '') ? undefined : Readable.toWeb(req)) + const request = new Request(url, { method, headers, body }) + + const response = await handler(request, env, ctx) + + res.statusCode = response.status + res.statusMessage = response.statusText + response.headers.forEach((v, k) => res.setHeader(k, v)) + if (!response.body) { + res.end() + return + } + + // @ts-expect-error + await pipeline(Readable.fromWeb(response.body), res) + } +} diff --git a/test/helpers/content-claims.js b/test/helpers/content-claims.js index af798a1..705e127 100644 --- a/test/helpers/content-claims.js +++ b/test/helpers/content-claims.js @@ -121,8 +121,9 @@ export const generateClaims = async (signer, dataCid, carCid, carStream, indexCi * @param {import('@ucanto/interface').Signer} signer * @param {import('multiformats').Link} shard * @param {ReadableStream} carStream CAR file data + * @param {URL} location */ -export const generateBlockLocationClaims = async (signer, shard, carStream) => { +export const generateBlockLocationClaims = async (signer, shard, carStream, location) => { /** @type {Claims} */ const claims = new LinkMap() @@ -131,7 +132,6 @@ export const generateBlockLocationClaims = async (signer, shard, carStream) => { .pipeTo(new WritableStream({ async write ({ cid, blockOffset, blockLength }) { const blocks = claims.get(cid) ?? [] - const location = new URL(`https://w3s.link/ipfs/${shard}?format=raw`) blocks.push(await generateLocationClaim(signer, shard, location, blockOffset, blockLength)) claims.set(cid, blocks) } @@ -155,8 +155,8 @@ export const generateLocationClaim = async (signer, content, location, offset, l nb: { content, location: [ - // @ts-expect-error string is not ${string}:$string - location.toString() + /** @type {import('@ucanto/interface').URI} */ + (location.toString()) ], range: { offset, length } } diff --git a/test/index.spec.js b/test/index.spec.js index 9b81c74..48357a0 100644 --- a/test/index.spec.js +++ b/test/index.spec.js @@ -14,6 +14,7 @@ import * as ByteRanges from 'byteranges' import { Builder, toBlobKey } from './helpers/builder.js' import { MAX_CAR_BYTES_IN_MEMORY } from '../src/constants.js' import { generateClaims, generateBlockLocationClaims, mockClaimsService, generateLocationClaim } from './helpers/content-claims.js' +import { mockBucketService } from './helpers/bucket.js' describe('freeway', () => { /** @type {Miniflare} */ @@ -22,6 +23,8 @@ describe('freeway', () => { let builder /** @type {import('./helpers/content-claims.js').MockClaimsService} */ let claimsService + /** @type {import('./helpers/bucket.js').MockBucketService} */ + let bucketService before(async () => { const bucketNames = ['CARPARK', 'SATNAV', 'DUDEWHERE'] @@ -47,15 +50,21 @@ describe('freeway', () => { }) const buckets = await Promise.all(bucketNames.map(b => miniflare.getR2Bucket(b))) - // @ts-expect-error + // @ts-expect-error bucket type mismatch builder = new Builder(buckets[0], buckets[1], buckets[2]) + // @ts-expect-error bucket type mismatch + bucketService = await mockBucketService(buckets[0]) }) beforeEach(() => { claimsService.resetCallCount() + bucketService.resetCallCount() }) - after(() => claimsService.close()) + after(() => { + claimsService.close() + bucketService.close() + }) it('should get a file', async () => { const input = randomBytes(256) @@ -99,6 +108,7 @@ describe('freeway', () => { const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${dataCid}/${input[0].path}`) + console.log(res.status, await res.text()) assert(!res.ok) assert.equal(res.status, 404) }) @@ -230,12 +240,11 @@ describe('freeway', () => { // no dudewhere or satnav so only content claims can satisfy the request const { dataCid, blobCids } = await builder.add(input, { asBlob: true }) - const carpark = await miniflare.getR2Bucket('CARPARK') - const res = await carpark.get(toBlobKey(blobCids[0].multihash)) - assert(res) + const location = new URL(toBlobKey(blobCids[0].multihash), bucketService.url) + const res = await fetch(location) + assert(res.body) - // @ts-expect-error nodejs ReadableStream does not implement ReadableStream interface correctly - const claims = await generateBlockLocationClaims(claimsService.signer, blobCids[0], res.body) + const claims = await generateBlockLocationClaims(claimsService.signer, blobCids[0], res.body, location) claimsService.setClaims(claims) const res1 = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${dataCid}/${input[0].path}`) @@ -355,8 +364,8 @@ describe('freeway', () => { const blobKey = toBlobKey(cid.multihash) await carpark.put(blobKey, input) - const url = new URL(`https://w3s.link/ipfs/${cid}?format=raw`) - const claim = await generateLocationClaim(claimsService.signer, cid, url, 0, input.length) + const location = new URL(blobKey, bucketService.url) + const claim = await generateLocationClaim(claimsService.signer, cid, location, 0, input.length) claimsService.setClaims(new LinkMap([[cid, [claim]]])) const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${cid}?format=raw`) @@ -380,8 +389,8 @@ describe('freeway', () => { const blobKey = toBlobKey(cid.multihash) await carpark.put(blobKey, input) - const url = new URL(`https://w3s.link/ipfs/${cid}?format=raw`) - const claim = await generateLocationClaim(claimsService.signer, cid, url, 0, input.length) + const location = new URL(blobKey, bucketService.url) + const claim = await generateLocationClaim(claimsService.signer, cid, location, 0, input.length) claimsService.setClaims(new LinkMap([[cid, [claim]]])) const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${cid}?format=raw`, { @@ -402,21 +411,18 @@ describe('freeway', () => { // no dudewhere or satnav so only content claims can satisfy the request const { blobCids } = await builder.add(input, { asBlob: true }) - const carpark = await miniflare.getR2Bucket('CARPARK') - const res0 = await carpark.get(toBlobKey(blobCids[0].multihash)) - assert(res0) - - const url = new URL(`https://w3s.link/ipfs/${blobCids[0]}?format=raw`) - const claim = await generateLocationClaim(claimsService.signer, blobCids[0], url, 0, input[0].content.length) + const location = new URL(toBlobKey(blobCids[0].multihash), bucketService.url) + const claim = await generateLocationClaim(claimsService.signer, blobCids[0], location, 0, input[0].content.length) claimsService.setClaims(new LinkMap([[blobCids[0], [claim]]])) - const res1 = await carpark.get(toBlobKey(blobCids[0].multihash)) - assert(res1) + const res = await fetch(location) + assert(res.ok) + assert(res.body) - await /** @type {ReadableStream} */ (res1.body) + await res.body .pipeThrough(new CARReaderStream()) .pipeTo(new WritableStream({ - async write ({ bytes, blockOffset, blockLength }) { + async write ({ cid, bytes, blockOffset, blockLength }) { const res = await miniflare.dispatchFetch(`http://localhost:8787/ipfs/${blobCids[0]}?format=raw`, { headers: { Range: `bytes=${blockOffset}-${blockOffset + blockLength - 1}` @@ -440,17 +446,17 @@ describe('freeway', () => { // no dudewhere or satnav so only content claims can satisfy the request const { blobCids } = await builder.add(input, { asBlob: true }) - const url = new URL(`https://w3s.link/ipfs/${blobCids[0]}?format=raw`) - const claim = await generateLocationClaim(claimsService.signer, blobCids[0], url, 0, input[0].content.length) + const location = new URL(toBlobKey(blobCids[0].multihash), bucketService.url) + const claim = await generateLocationClaim(claimsService.signer, blobCids[0], location, 0, input[0].content.length) claimsService.setClaims(new LinkMap([[blobCids[0], [claim]]])) - const carpark = await miniflare.getR2Bucket('CARPARK') - const res0 = await carpark.get(toBlobKey(blobCids[0].multihash)) - assert(res0) + const res0 = await fetch(location) + assert(res0.ok) + assert(res0.body) /** @type {Array} */ const blocks = [] - await /** @type {ReadableStream} */ (res0.body) + await res0.body .pipeThrough(new CARReaderStream()) .pipeTo(new WritableStream({ write (block) { blocks.push(block) } }))