gateway: remote car gateway backend (#587)

ipfs · Apr 13, 2024 · a26b503 · a26b503
1 parent 63c3ec6
commit a26b503
Show file tree

Hide file tree

Showing 33 changed files with 3,956 additions and 267 deletions.
diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml
@@ -1,17 +1,23 @@
 name: Gateway Conformance
+# This workflow runs https://github.com/ipfs/gateway-conformance
+# against different backend implementations of boxo/gateway
 
 on:
   push:
     branches:
       - main
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
   cancel-in-progress: true
 
 jobs:
-  gateway-conformance:
+  # This test uses a static CAR file as a local blockstore,
+  # allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go)
+  # which is used by implementations like Kubo
+  local-block-backend:
     runs-on: ubuntu-latest
     steps:
       # 1. Download the gateway-conformance fixtures
@@ -21,49 +27,185 @@ jobs:
           output: fixtures
           merged: true
 
-      # 2. Build the car-gateway
+      # 2. Build the gateway binary
+      - name: Checkout boxo
+        uses: actions/checkout@v4
+        with:
+          path: boxo
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build test-gateway
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/car-file
+
+      # 3. Start the gateway binary
+      - name: Start test-gateway
+        run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 &
+
+      # 4. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.5
+        with:
+          gateway-url: http://127.0.0.1:8040
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          subdomain-url: http://example.net
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
+
+      # 5. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_local-block-backend.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_local-block-backend.json
+          path: output.json
+
+  # This test uses remote block gateway (?format=raw) as a remote blockstore,
+  # allowing us to test conformance against RemoteBlocksBackend
+  # (gateway/backend_blocks.go) which is used by implementations like
+  # rainbow configured to use with remote block backend
+  # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
+  remote-block-backend:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5
+        with:
+          output: fixtures
+          merged: true
+
+      # 2. Build the gateway binaries
+      - name: Checkout boxo
+        uses: actions/checkout@v4
+        with:
+          path: boxo
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build remote-block-backend # it will act as a trustless CAR gateway
+        run: go build -o remote-block-backend
+        working-directory: boxo/examples/gateway/car-file
+      - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/proxy-blocks
+
+      # 3. Start the gateway binaries
+      - name: Start remote HTTP backend that serves application/vnd.ipld.raw
+        run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests
+      - name: Start gateway that uses the remote block backend
+        run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 &
+
+      # 4. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.5
+        with:
+          gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          subdomain-url: http://example.net
+          specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
+          args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
+
+      # 5. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
         with:
-          go-version: 1.21.x
+          name: gateway-conformance_remote-block-backend.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: gateway-conformance_remote-block-backend.json
+          path: output.json
+
+  # This test uses remote CAR gateway (?format=car, IPIP-402)
+  # allowing us to test conformance against remote CarFetcher backend.
+  # (gateway/backend_car_fetcher.go) which is used by implementations like
+  # rainbow configured to use with remote car backend
+  # Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car
+  remote-car-backend:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5
+        with:
+          output: fixtures
+          merged: true
+
+      # 2. Build the gateway binaries
       - name: Checkout boxo
         uses: actions/checkout@v4
         with:
           path: boxo
-      - name: Build car-gateway
-        run: go build -o car-gateway
-        working-directory: boxo/examples/gateway/car
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'boxo/examples/go.mod'
+          cache-dependency-path: "boxo/**/*.sum"
+      - name: Build remote-car-backend # it will act as a trustless CAR gateway
+        run: go build -o remote-car-backend
+        working-directory: boxo/examples/gateway/car-file
+      - name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend
+        run: go build -o test-gateway
+        working-directory: boxo/examples/gateway/proxy-car
 
-      # 3. Start the car-gateway
-      - name: Start car-gateway
-        run: boxo/examples/gateway/car/car-gateway -c fixtures/fixtures.car -p 8040 &
+      # 3. Start the gateway binaries
+      - name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402)
+        run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests
+      - name: Start gateway that uses the remote CAR backend
+        run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 &
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests
         uses: ipfs/gateway-conformance/.github/actions/test@v0.5
         with:
-          gateway-url: http://127.0.0.1:8040
+          gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
           json: output.json
           xml: output.xml
           html: output.html
           markdown: output.md
           subdomain-url: http://example.net
           specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
-          args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
+          args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
 
       # 5. Upload the results
       - name: Upload MD summary
         if: failure() || success()
         run: cat output.md >> $GITHUB_STEP_SUMMARY
       - name: Upload HTML report
         if: failure() || success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.html
+          name: gateway-conformance_remote-car-backend.html
           path: output.html
       - name: Upload JSON report
         if: failure() || success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: gateway-conformance.json
+          name: gateway-conformance_remote-car-backend.json
           path: output.json
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -16,7 +16,9 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
-* `gateway` now includes `NewRemoteBlocksBackend` which allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced a `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
+* ✨ `gateway` has new backend possibilities:
+  * `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
+  * `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
 
 ### Changed
 

diff --git a/examples/README.md b/examples/README.md
@@ -27,6 +27,7 @@ Once you have your example finished, do not forget to run `go mod tidy` and addi
 ## Examples and Tutorials
 
 - [Fetching a UnixFS file by CID](./unixfs-file-cid)
-- [Gateway backed by a CAR file](./gateway/car)
-- [Gateway backed by a remote blockstore and IPNS resolver](./gateway/proxy)
+- [Gateway backed by a local blockstore in form of a CAR file](./gateway/car-file)
+- [Gateway backed by a remote (HTTP) blockstore and IPNS resolver](./gateway/proxy-blocks)
+- [Gateway backed by a remote (HTTP) CAR Gateway](./gateway/proxy-car)
 - [Delegated Routing V1 Command Line Client](./routing/delegated-routing-client/)
diff --git a/examples/gateway/car/README.md → examples/gateway/car-file/README.md b/examples/gateway/car/README.md → examples/gateway/car-file/README.md
@@ -1,13 +1,16 @@
-# HTTP Gateway backed by a CAR File
+# HTTP Gateway backed by a CAR File as BlocksBackend
 
 This is an example that shows how to build a Gateway backed by the contents of
 a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content
 Addressable aRchive that contains blocks.
 
+The `main.go` sets up a `blockService` backed by a static CAR file,
+and then uses it to initialize `gateway.NewBlocksBackend(blockService)`.
+
 ## Build
 
 ```bash
-> go build -o car-gateway
+> go build -o gateway
 ```
 
 ## Usage
@@ -23,7 +26,7 @@ Then, you can start the gateway with:
 
 
 ```
-./car-gateway -c data.car -p 8040
+./gateway -c data.car -p 8040
 ```
 
 ### Subdomain gateway

diff --git a/examples/gateway/car/main.go → examples/gateway/car-file/main.go b/examples/gateway/car/main.go → examples/gateway/car-file/main.go
diff --git a/examples/gateway/car/main_test.go → examples/gateway/car-file/main_test.go b/examples/gateway/car/main_test.go → examples/gateway/car-file/main_test.go
diff --git a/examples/gateway/car/test.car → examples/gateway/car-file/test.car b/examples/gateway/car/test.car → examples/gateway/car-file/test.car
diff --git a/examples/gateway/proxy/README.md → examples/gateway/proxy-blocks/README.md b/examples/gateway/proxy/README.md → examples/gateway/proxy-blocks/README.md
@@ -18,7 +18,7 @@ gateway using `?format=ipns-record`. In addition, DNSLink lookups are done local
 ## Build
 
 ```bash
-> go build -o verifying-proxy
+> go build -o gateway
 ```
 
 ## Usage
@@ -28,7 +28,7 @@ types. Once you have it, run the proxy gateway with its address as the host para
 
 
 ```
-./verifying-proxy -g https://ipfs.io -p 8040
+./gateway -g https://trustless-gateway.link -p 8040
 ```
 
 ### Subdomain gateway

diff --git a/examples/gateway/proxy/main.go → examples/gateway/proxy-blocks/main.go b/examples/gateway/proxy/main.go → examples/gateway/proxy-blocks/main.go
@@ -28,7 +28,7 @@ func main() {
 	defer (func() { _ = tp.Shutdown(ctx) })()
 
 	// Creates the gateway with the remote block store backend.
-	backend, err := gateway.NewRemoteBlocksBackend([]string{*gatewayUrlPtr})
+	backend, err := gateway.NewRemoteBlocksBackend([]string{*gatewayUrlPtr}, nil)
 	if err != nil {
 		log.Fatal(err)
 	}

diff --git a/examples/gateway/proxy/main_test.go → examples/gateway/proxy-blocks/main_test.go b/examples/gateway/proxy/main_test.go → examples/gateway/proxy-blocks/main_test.go
@@ -21,7 +21,7 @@ const (
 )
 
 func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server {
-	backend, err := gateway.NewRemoteBlocksBackend([]string{rs.URL})
+	backend, err := gateway.NewRemoteBlocksBackend([]string{rs.URL}, nil)
 	require.NoError(t, err)
 	handler := common.NewHandler(backend)
 	ts := httptest.NewServer(handler)

diff --git a/examples/gateway/proxy-car/README.md b/examples/gateway/proxy-car/README.md
@@ -0,0 +1,51 @@
+# Gateway as Proxy for Trustless CAR Remote Backend
+
+This is an example of building a "verifying proxy" Gateway that has no
+local on-disk blockstore, but instead, uses `application/vnd.ipld.car` and
+`application/vnd.ipfs.ipns-record` responses from a remote HTTP server that
+implements CAR support from [Trustless Gateway
+Specification](https://specs.ipfs.tech/http-gateways/trustless-gateway/).
+
+**NOTE:** the remote CAR backend MUST implement [IPIP-0402: Partial CAR Support on Trustless Gateways](https://specs.ipfs.tech/ipips/ipip-0402/)
+
+## Build
+
+```bash
+> go build -o gateway
+```
+
+## Usage
+
+First, you need a compliant gateway that supports both [CAR requests](https://www.iana.org/assignments/media-types/application/vnd.ipld.car) and IPNS Record response
+types. Once you have it, run the proxy gateway with its address as the host parameter:
+
+```
+./gateway -g https://trustless-gateway.link -p 8040
+```
+
+### Subdomain gateway
+
+Now you can access the gateway in [`localhost:8040`](http://localhost:8040/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze). It will
+behave like a regular [subdomain gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#subdomain-gateway),
+except for the fact that it runs no libp2p, and has no local blockstore.
+All data is provided by a remote trustless gateway, fetched as CAR files and IPNS Records, and verified locally.
+
+### Path gateway
+
+If you don't need Origin isolation and only care about hosting flat files,
+a plain [path gateway](https://docs.ipfs.tech/how-to/address-ipfs-on-web/#path-gateway) at
+[`127.0.0.1:8040`](http://127.0.0.1:8040/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi)
+may suffice.
+
+### DNSLink gateway
+
+Gateway supports hosting of [DNSLink](https://dnslink.dev/) websites. All you need is to pass `Host` header with FQDN that has DNSLink set up:
+
+```console
+$ curl -sH 'Host: en.wikipedia-on-ipfs.org' 'http://127.0.0.1:8080/wiki/' | head -3
+<!DOCTYPE html><html class="client-js"><head>
+  <meta charset="UTF-8">
+  <title>Wikipedia, the free encyclopedia</title>
+```
+
+Put it behind a reverse proxy terminating TLS (like Nginx) and voila!