feat: improvements & gateway-conformance support (light) (#57)

Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>

.env-all

@@ -0,0 +1,18 @@
+# ENV vars recommended for running with everything enabled:
+# * trustless gateways
+# * delegated routing
+# * libp2p
+# * bitswap
+USE_LIBP2P=true
+USE_BITSWAP=true
+DEBUG='helia-http-gateway*,*helia-fetch*'
+USE_TRUSTLESS_GATEWAYS=true
+USE_DELEGATED_ROUTING=true
+
+# Uncomment the two below to save blockstore and datastore to disk
+# FILE_DATASTORE_PATH=./data/datastore
+# FILE_BLOCKSTORE_PATH=./data/blockstore
+
+# If you want to run the gateway with a local trustless gateway, uncomment the below
+# PORT=8090 # or whatever port you want to run helia-http-gateway on
+# TRUSTLESS_GATEWAYS=http://127.0.0.1:8080

.env-delegated-routing

@@ -0,0 +1,16 @@
+# ENV vars recommended for running with only delegated-routing-v1-http
+USE_LIBP2P=false
+# needed to request blocks from peers we discover with delegated routing
+USE_BITSWAP=true
+# TRUSTLESS_GATEWAYS=http://127.0.0.1:8080
+DEBUG='helia-http-gateway*,*helia-fetch*,*delegated-routing-v1-http-api-client*'
+USE_TRUSTLESS_GATEWAYS=false
+USE_DELEGATED_ROUTING=true
+
+# IF you're delegating to kubo running locally you should uncomment the two below:
+# PORT=8090
+# DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080
+
+# Uncomment the two below to save blockstore and datastore to disk
+# FILE_DATASTORE_PATH=./data/datastore
+# FILE_BLOCKSTORE_PATH=./data/blockstore

.env-gwc

@@ -0,0 +1,16 @@
+# ENV vars recommended for running gateway-conformance tests
+USE_LIBP2P=false
+USE_BITSWAP=true
+PORT=8090 # helia-http-gateway should be running here
+TRUSTLESS_GATEWAYS=http://127.0.0.1:8080 # Kubo should be running here
+DELEGATED_ROUTING_V1_HOST=http://127.0.0.1:8080 # Kubo should be running here
+DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*'
+USE_TRUSTLESS_GATEWAYS=true
+USE_DELEGATED_ROUTING=true
+
+# Uncomment the two below to save blockstore and datastore to disk
+# FILE_DATASTORE_PATH=./data/datastore
+# FILE_BLOCKSTORE_PATH=./data/blockstore
+
+# Uncomment the below to see request & response headers in the logs
+# ECHO_HEADERS=true

.env-trustless-only

@@ -0,0 +1,14 @@
+# ENV vars recommended for running with trustless gateways only
+USE_LIBP2P=false
+USE_BITSWAP=false
+DEBUG='helia-http-gateway*,*helia-fetch*,*helia:trustless-gateway-block-broker*'
+USE_TRUSTLESS_GATEWAYS=true
+USE_DELEGATED_ROUTING=false
+
+# Uncomment the two below to save blockstore and datastore to disk
+# FILE_DATASTORE_PATH=./data/datastore
+# FILE_BLOCKSTORE_PATH=./data/blockstore
+
+# If you want to run the gateway with a local trustless gateway, uncomment the below
+# PORT=8090 # or whatever port you want to run helia-http-gateway on
+# TRUSTLESS_GATEWAYS=http://127.0.0.1:8080

.github/workflows/gateway-conformance.yml

@@ -0,0 +1,121 @@
+name: Gateway Conformance
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  gateway-conformance:
+    runs-on: ubuntu-latest
+    steps:
+      # 1. Start the Kubo gateway
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.21.x
+
+      - name: Install Kubo gateway from source
+        #uses: ipfs/download-ipfs-distribution-action@v1
+        run: |
+          go install github.com/ipfs/kubo/cmd/ipfs@v0.24.0
+      - name: Setup kubo config
+        run: |
+          ipfs init --profile=test
+          ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/8080"
+
+      # 2. Download the gateway-conformance fixtures
+      - name: Download gateway-conformance fixtures
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.4.1
+        with:
+          output: fixtures
+
+      - name: Start Kubo gateway
+        uses: ipfs/start-ipfs-daemon-action@v1
+
+      # 3. Populate the Kubo gateway with the gateway-conformance fixtures
+      - name: Import fixtures
+        run: |
+          # Import car files
+          find ./fixtures -name '*.car' -exec ipfs dag import --pin-roots=false --offline {} \;
+
+          # Import ipns records
+          records=$(find ./fixtures -name '*.ipns-record')
+          for record in $records
+          do
+              key=$(basename -s .ipns-record "$record" | cut -d'_' -f1)
+              ipfs routing put --allow-offline "/ipns/$key" "$record"
+          done
+
+          # Import dnslink records
+          # the IPFS_NS_MAP env will be used by the daemon
+          export IPFS_NS_MAP=$(cat "./fixtures/dnslinks.json" | jq -r '.subdomains | to_entries | map("\(.key).example.com:\(.value)") | join(",")')
+          export IPFS_NS_MAP="$(cat "./fixtures/dnslinks.json" | jq -r '.domains | to_entries | map("\(.key):\(.value)") | join(",")'),${IPFS_NS_MAP}"
+          echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV
+
+      # 4. Build helia-http-gateway
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
+      - name: Checkout helia-http-gateway
+        uses: actions/checkout@v3
+        with:
+          path: helia-http-gateway
+
+      - name: Install dependencies
+        run: npm ci
+        working-directory: helia-http-gateway
+
+      - name: Build helia-http-gateway
+        run: npm run build
+        working-directory: helia-http-gateway
+
+      # 5. Start helia-http-gateway
+      - name: Start helia-http-gateway
+        env:
+          GATEWAY_CONFORMANCE_TEST: true
+          TRUSTLESS_GATEWAYS: "http://127.0.0.1:8080"
+          USE_LIBP2P: false
+          PORT: 8090
+        run: |
+          # run gw
+          node dist/src/index.js &
+        working-directory: helia-http-gateway
+
+      # 6. Run the gateway-conformance tests
+      - name: Run gateway-conformance tests
+        uses: ipfs/gateway-conformance/.github/actions/test@v0.4.1
+        with:
+          gateway-url: http://127.0.0.1:8090
+          subdomain-url: http://127.0.0.1:8090
+          json: output.json
+          xml: output.xml
+          html: output.html
+          markdown: output.md
+          # specs: subdomain-ipfs-gateway,subdomain-ipns-gateway
+          # use below to skip specific test if needed
+          # args: -skip 'TestFooBr/GET_response_for_something'
+          #
+          # only-if-cached: helia-ht does not guarantee local cache, we will adjust upstream test (which was Kubo-specific)
+          # for now disabling these test cases
+          args: -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|proxy|TestGatewaySubdomainAndIPNS|TestGatewaySubdomains|Trustless|TestGatewayIPNSRecord|RedirectsFile|TestGatewayUnixFSFileRanges|TestGatewayJSONCborAndIPNS|TestTar|Symlink|TestPathGatewayMiscellaneous|TestGatewayBlock|TestRedirectCanonicalIPNS|TestGatewayIPNSPath|TestNativeDag|TestPathing|TestPlainCodec|TestDagPbConversion|TestGatewayJsonCbor|TestCors).*$'
+
+      # 7. Upload the results
+      - name: Upload MD summary
+        if: failure() || success()
+        run: cat output.md >> $GITHUB_STEP_SUMMARY
+      - name: Upload HTML report
+        if: failure() || success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance.html
+          path: output.html
+      - name: Upload JSON report
+        if: failure() || success()
+        uses: actions/upload-artifact@v3
+        with:
+          name: gateway-conformance.json
+          path: output.json

.github/workflows/playwright.yml

@@ -19,6 +19,25 @@ jobs:
       run: npm ci
     - name: Install Playwright Browsers
       run: npx playwright install --with-deps
+
+    # Cache playwright binaries
+    - uses: actions/cache@v3
+      id: playwright-cache
+      with:
+        path: |
+          ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }}
+    - run: npx playwright install --with-deps
+      if: steps.playwright-cache.outputs.cache-hit != 'true'
+
+    # Cache datastores
+    - uses: actions/cache@v3
+      id: e2e-datastore-and-blockstore
+      with:
+        path: |
+          ./test/fixtures/e2e
+        key: ${{ runner.os }}-e2e-stores-${{ hashFiles('**/package-lock.json') }}
+
     - name: Run Playwright tests
       run: npm run test:e2e
       env:

.gitignore

@@ -134,3 +134,6 @@ build
 /playwright/.cache/
 screenshots
 .envrc
+scripts/tmp/fixtures
+scripts/tmp/kubo-path.txt
+test/fixtures/e2e

DEVELOPER-NOTES.md

@@ -0,0 +1,84 @@
+# Developer notes
+
+<!-- toc -->
+
+## Gateway Conformance testing
+
+We have some code enabled that makes running gateway-conformance testing against helia-http-gateway easy. Follow the instructions in this section to run gateway-conformance tests locally
+
+### Prerequisites
+
+1. [Install docker](https://docs.docker.com/get-docker/)
+2. [Install nodejs](https://nodejs.org/en/download/)
+
+### Run gateway-conformance tests locally (once)
+
+```sh
+$ npm run test:gwc
+```
+
+### Continuously develop while running gateway-conformance tests
+
+```sh
+# terminal 1
+$ npm run test:gwc-kubo
+# You can also set up the kubo backing node with the instructions at https://github.com/ipfs/gateway-conformance/blob/main/docs/development.md#developing-against-kubo
+
+# terminal 2
+# you will need to stop and start this one in-between code changes. It's not watching for changes
+$ DEBUG="helia-http-gateway*" FILE_DATASTORE_PATH=./tmp/datastore npm run test:gwc-helia
+
+# terminal 3
+$ npm run test:gwc-execute
+# To skip some tests, run something like:
+# npm run test:gwc-execute -- -skip '^.*(DirectoryListing|TestGatewayCache|TestSubdomainGatewayDNSLinkInlining|TestGatewaySubdomainAndIPNS).*$'
+
+# OR from the gateway-conformance repo directly with something like:
+go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m
+
+```
+
+
+
+### Some callouts
+
+1. You may want to set up an nginx proxy from `http://helia-http-gateway.localhost` to `http://localhost:8090`to help with the gateway-conformance tests. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185)
+1. You may want to run the gateway-conformance tests directly from the repo if you're on a macOS M1 due to some issues with docker and the proxying that the gateway-conformance testing tool uses. If you do this, you'll need to run `make gateway-conformance` in the `gateway-conformance` repo root, and then run the tests with something like `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --specs subdomain-ipfs-gateway,subdomain-ipns-gateway --json gwc-report.json -- -timeout 30m`.
+    - If you want to run a specific test, you can pass the `-run` gotest flag. e.g. `go run ./cmd/gateway-conformance/main.go test --gateway-url 'http://localhost:8090' --subdomain-url 'http://localhost:8090' --json gwc-report.json -- -timeout 30m -run 'TestGatewaySubdomains/request_for_example.com%2Fipfs%2F%7BCIDv1%7D_redirects_to_subdomain_%28HTTP_proxy_tunneling_via_CONNECT%29#01'`
+1. The file `./scripts/kubo-init.js` executes kubo using `execa` instead of `ipfsd-ctl` so there may be some gotchas, but it should be as cross-platform and stable as the `execa` library.
+1. The IPFS_PATH used is a temporary directory. Your OS should handle removing it when vital, but you can also remove it manually. The path to this directory is printed out when the tests start, and saved in a file at `./scripts/tmp/kubo-path.txt`.
+1. The tests save gateway-conformance fixtures to `./scripts/tmp/fixtures`. You can remove this directory manually if you want to re-run the tests with a fresh set of fixtures.
+1. The results of the gateway-conformance tests are saved to `./gwc-report.json`. This file is overwritten every time the tests are run.
+1. The gateway-conformance tests are flaky and commands & documentation are not up to date. Running commands with CLI flags is supposed to work, and env vars aren't documented, but ENV_VARs are the only way to get the tests to run, but not when ran with docker. See [this issue](https://github.com/ipfs/gateway-conformance/issues/185#issuecomment-1839801366)
+
+
+## Tiros info
+
+### Deploying to Tiros
+
+Go to https://github.com/plprobelab/probelab-infra/blob/main/aws/tf/tiros.tf
+
+update helia stuff
+
+run terraform apply (with AWS Config)
+
+### Kick off helia-http-gateway task manually:
+
+1. https://us-west-1.console.aws.amazon.com/ecs/v2/clusters/prod-usw1-ecs-cluster/run-task?region=us-west-1
+1. select launch type FARGATE
+1. select deployment configuration application type to be task
+1. select task-helia family
+1. select vpc (non-default)
+1. select security group for tiros
+1. Click create
+
+### Todo
+
+- [ ] Fix helia_health_cmd (https://github.com/plprobelab/probelab-infra/blob/7ec47f16e84113545cdb06b07f865a3bc5787a0b/aws/tf/tiros.tf#L5C4-L5C4) to use the new helia-health command
+- [ ] fix node max memory for helia-http-gateway docker container to support at least 4GB
+
+### Links
+
+graphs for runs are at https://probelab.grafana.net/d/GpwxraxVk/tiros-ops?orgId=1&from=now-7d&to=now&inspect=8&inspectTab=error
+AWS logs for runs are at https://us-west-1.console.aws.amazon.com/cloudwatch/home?region=us-west-1#logsV2:log-groups/log-group/prod-usw1-cmi-tiros-ipfs (you need to be logged in to AWS)
+

README.md

@@ -53,7 +53,8 @@ $ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway*" helia-http-gateway:
 | `USE_TRUSTLESS_GATEWAYS` | Whether to fetch content from trustless-gateways or not | `true` |
 | `TRUSTLESS_GATEWAYS` | Comma separated list of trusted gateways to fetch content from | [Defined in Helia](https://github.com/ipfs/helia/blob/main/packages/helia/src/block-brokers/trustless-gateway/index.ts) |
 | `USE_LIBP2P` | Whether to use libp2p networking | `true` |
-| `RESOLVE_REDIRECTS` | Whether to resolve redirects before looking up dnslink entries | `true` |
+| `ECHO_HEADERS` | A debug flag to indicate whether you want to output request and response headers | `false` |
+| `DELEGATED_ROUTING_V1_HOST` | Hostname to use for delegated routing v1 | `https://delegated-ipfs.dev` |
 
 <!--
 TODO: currently broken when used in docker, but they work when running locally (you can cache datastore and blockstore locally to speed things up if you want)
@@ -63,6 +64,13 @@ TODO: currently broken when used in docker, but they work when running locally (
 
 See the source of truth for all `process.env.<name>` environment variables at [src/constants.ts](src/constants.ts).
 
+You can also see some recommended environment variable configurations at:
+
+- [./.env-all](./.env-all)
+- [./.env-delegated-routing](./.env-delegated-routing)
+- [./.env-gwc](./.env-gwc)
+- [./.env-trustless-only](./.env-trustless-only)
+
 ### Running with custom configurations
 
 Note that any of the following calls to docker can be replaced with something like `MY_ENV_VAR="MY_VALUE" npm run start`

e2e-tests/smoketest.spec.ts

@@ -9,7 +9,7 @@ const pages = [
   'docs.ipfs.tech',
   'docs.libp2p.io',
   'drand.love',
-  'fil.org',
+  // 'fil.org', // timing out
   // 'filecoin.io', // timing out
   // 'green.filecoin.io', // timing out
   'ipfs.tech',
@@ -36,7 +36,7 @@ test.beforeEach(async ({ context }) => {
 })
 
 test.afterEach(async ({ page }) => {
-  test.setTimeout(30000)
+  test.setTimeout(2 * 60 * 1000) // 2 minutes
   const result = await page.request.post(`http://localhost:${PORT}/api/v0/repo/gc`)
   expect(result?.status()).toBe(200)