feat: remove dudewhere and satnav (#110)

Content claims should satisfy ALL requests now. If no content claims,
then we genuinely do not have, with one exception (see below). Hence
falling back to DUDEWHERE/SATNAV does not yield any results.

This removes DUDEWHERE and SATNAV bindings but keeps CARPARK to allow us
to serve CAR files from R2 - not all CAR files (legacy CARs) have
content claims.

.github/actions/test/action.yml

@@ -7,7 +7,7 @@ runs:
     - uses: actions/setup-node@v3
       with:
         registry-url: 'https://registry.npmjs.org'
-        node-version: 18
+        node-version: 20
         cache: 'npm'
     - run: npm ci
       shell: bash

package.json

@@ -9,10 +9,6 @@
     ".": {
       "import": "./src/index.js",
       "types": "./dist/src/index.d.ts"
-    },
-    "./blockstore": {
-      "import": "./src/lib/blockstore.js",
-      "types": "./dist/src/lib/blockstore.d.ts"
     }
   },
   "scripts": {
@@ -36,37 +32,26 @@
   "author": "Alan Shaw",
   "license": "Apache-2.0 OR MIT",
   "dependencies": {
-    "@ipld/car": "^5.2.6",
-    "@ipld/dag-cbor": "^9.0.8",
-    "@ipld/dag-json": "^10.1.7",
-    "@ipld/dag-pb": "^4.0.8",
     "@web3-storage/blob-fetcher": "^2.2.0",
     "@web3-storage/gateway-lib": "^5.0.1",
-    "cardex": "^3.0.0",
     "dagula": "^8.0.0",
     "http-range-parse": "^1.0.0",
     "lnmap": "^2.0.0",
-    "multiformats": "^13.0.1",
-    "p-defer": "^4.0.0"
+    "multiformats": "^13.0.1"
   },
   "devDependencies": {
-    "@aws-sdk/client-s3": "^3.490.0",
     "@cloudflare/workers-types": "^4.20231218.0",
     "@ucanto/principal": "^8.1.0",
     "@web3-storage/content-claims": "^5.0.0",
     "@web3-storage/public-bucket": "^1.1.0",
-    "ava": "^5.3.1",
-    "carbites": "^1.0.6",
+    "@web3-storage/upload-client": "^16.1.1",
     "carstream": "^2.1.0",
-    "dotenv": "^16.3.1",
     "esbuild": "^0.18.20",
     "files-from-path": "^0.2.6",
-    "ipfs-car": "^0.9.2",
-    "miniflare": "^2.14.1",
+    "miniflare": "^3.20240610.1",
     "multipart-byte-range": "^3.0.1",
     "standard": "^17.1.0",
-    "typescript": "^5.3.3",
-    "uint8arrays": "^4.0.10"
+    "typescript": "^5.3.3"
   },
   "standard": {
     "ignore": [

src/bindings.d.ts

@@ -1,47 +1,7 @@
-import type { Link } from 'multiformats/link'
-import type { Context } from '@web3-storage/gateway-lib'
-import type { CARLink } from 'cardex/api'
-import type { R2Bucket, KVNamespace } from '@cloudflare/workers-types'
-import type { MemoryBudget } from './lib/mem-budget'
-
-export {}
+import type { R2Bucket } from '@cloudflare/workers-types'
 
 export interface Environment {
   DEBUG: string
   CARPARK: R2Bucket
-  DUDEWHERE: R2Bucket
-  SATNAV: R2Bucket
-  MAX_SHARDS: string
   CONTENT_CLAIMS_SERVICE_URL?: string
 }
-
-/**
- * Simple bucket does not allow range requests or support metadata on returned
- * objects.
- */
-export interface SimpleBucket {
-  get (key: string): Promise<SimpleBucketObject | null>
-}
-
-export interface SimpleBucketObject {
-  readonly key: string
-  readonly body: ReadableStream
-  arrayBuffer(): Promise<ArrayBuffer>
-}
-
-export interface IndexSource {
-  /** Bucket this index can be found in */
-  bucket: SimpleBucket
-  /** Bucket key for the source */
-  key: string
-  /**
-   * Origin CAR CID the index source applies to. Will be undefined if the index
-   * source is a multi-index index, which specifies origin CAR CIDs within the
-   * index.
-   */
-  origin?: CARLink
-}
-
-export interface IndexSourcesContext extends Context {
-  indexSources: IndexSource[]
-}

src/constants.js

@@ -1,2 +1 @@
-export const MAX_CAR_BYTES_IN_MEMORY = 1024 * 1024 * 5
 export const CAR_CODE = 0x0202

src/handlers/car-block.js

@@ -1,10 +1,15 @@
 /* eslint-env browser */
 /* global FixedLengthStream */
 import { HttpError } from '@web3-storage/gateway-lib/util'
+// @ts-expect-error no types
+import httpRangeParse from 'http-range-parse'
+import { base58btc } from 'multiformats/bases/base58'
 import { CAR_CODE } from '../constants.js'
-import * as Http from '../lib/http.js'
 
-/** @typedef {import('@web3-storage/gateway-lib').IpfsUrlContext} CarBlockHandlerContext */
+/**
+ * @typedef {import('@web3-storage/gateway-lib').IpfsUrlContext} CarBlockHandlerContext
+ * @typedef {{ offset: number, length?: number } | { offset?: number, length: number } | { suffix: number }} Range
+ */
 
 /**
  * Handler that serves CAR files directly from R2.
@@ -29,7 +34,10 @@ export async function handleCarBlock (request, env, ctx) {
   }
 
   if (request.method === 'HEAD') {
-    const obj = await env.CARPARK.head(`${dataCid}/${dataCid}.car`)
+    let obj = await env.CARPARK.head(toBlobKey(dataCid.multihash))
+    if (!obj) {
+      obj = await env.CARPARK.head(toCARKey(dataCid))
+    }
     if (!obj) throw new HttpError('CAR not found', { status: 404 })
     return new Response(undefined, {
       headers: {
@@ -40,17 +48,20 @@ export async function handleCarBlock (request, env, ctx) {
     })
   }
 
-  /** @type {import('../lib/http.js').Range|undefined} */
+  /** @type {Range|undefined} */
   let range
   if (request.headers.has('range')) {
     try {
-      range = Http.parseRange(request.headers.get('range') ?? '')
+      range = parseRange(request.headers.get('range') ?? '')
     } catch (err) {
       throw new HttpError('invalid range', { status: 400, cause: err })
     }
   }
 
-  const obj = await env.CARPARK.get(`${dataCid}/${dataCid}.car`, { range })
+  let obj = await env.CARPARK.get(toBlobKey(dataCid.multihash), { range })
+  if (!obj) {
+    obj = await env.CARPARK.get(toCARKey(dataCid), { range })
+  }
   if (!obj) throw new HttpError('CAR not found', { status: 404 })
 
   const status = range ? 206 : 200
@@ -80,3 +91,27 @@ export async function handleCarBlock (request, env, ctx) {
   // @ts-expect-error ReadableStream types incompatible
   return new Response(obj.body.pipeThrough(new FixedLengthStream(contentLength)), { status, headers })
 }
+
+/** @param {import('multiformats').UnknownLink} cid */
+const toCARKey = cid => `${cid}/${cid}.car`
+
+/** @param {import('multiformats').MultihashDigest} digest */
+const toBlobKey = digest => {
+  const mhStr = base58btc.encode(digest.bytes)
+  return `${mhStr}/${mhStr}.blob`
+}
+
+/**
+ * Convert a HTTP Range header to a range object.
+ * @param {string} value
+ * @returns {Range}
+ */
+function parseRange (value) {
+  const result = httpRangeParse(value)
+  if (result.ranges) throw new Error('Multipart ranges not supported')
+  const { unit, first, last, suffix } = result
+  if (unit !== 'bytes') throw new Error(`Unsupported range unit: ${unit}`)
+  return suffix != null
+    ? { suffix }
+    : { offset: first, length: last != null ? last - first + 1 : undefined }
+}