refactor: hamt dir and range test as go test

test/cli/fixtures/README.md

@@ -0,0 +1,72 @@
+# Dataset Description / Sources
+
+TestGatewayHAMTDirectory.car generated with:
+
+```bash
+ipfs version
+# ipfs version 0.19.0
+
+export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm
+export IPFS_PATH=$(mktemp -d)
+
+# Init and start daemon, ensure we have an empty repository.
+ipfs init --empty-repo
+ipfs daemon &> /dev/null &
+export IPFS_PID=$!
+
+# Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon.
+curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/
+kill $IPFS_PID
+
+# Get the list with all the downloaded refs and sanity check.
+ipfs refs local > required_refs
+cat required_refs | wc -l
+# 962
+
+# Get the list of all the files CIDs inside the directory and sanity check.
+cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs
+cat files_refs | wc -l
+# 10100
+
+# Make and export our fixture.
+ipfs files mkdir --cid-version 1 /fixtures
+cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
+cat files_refs | ipfs files write --create /fixtures/files_refs
+export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
+echo $FIXTURE_CID
+# bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i
+ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car
+```
+
+TestGatewayMultiRange.car generated with:
+
+
+```sh
+ipfs version
+# ipfs version 0.19.0
+
+export FILE_CID=bafybeibkzwf3ffl44yfej6ak44i7aly7rb4udhz5taskreec7qemmw5jiu
+export IPFS_PATH=$(mktemp -d)
+
+# Init and start daemon, ensure we have an empty repository.
+ipfs init --empty-repo
+ipfs daemon &> /dev/null &
+export IPFS_PID=$!
+
+# Get a specific byte range from the file. 
+curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=2000-2002, 40000000000-40000000002"
+kill $IPFS_PID
+
+# Get the list with all the downloaded refs and sanity check.
+ipfs refs local > required_refs
+cat required_refs | wc -l
+# 48
+
+# Make and export our fixture.
+ipfs files mkdir --cid-version 1 /fixtures
+cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
+export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
+echo $FIXTURE_CID
+# bafybeihqs4hdx64a6wmrclp3a2pwxkd5prwdos45bdftpegls5ktzspi7a
+ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car
+```

test/cli/gateway_range_test.go

@@ -0,0 +1,116 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ipfs/kubo/test/cli/harness"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGatewayHAMTDirectory(t *testing.T) {
+	t.Parallel()
+
+	const (
+		// The CID of the HAMT-sharded directory that has 10k items
+		hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm"
+
+		// fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG
+		// representing the minimal set of blocks necessary for directory listing.
+		// It also includes a "files_refs" file with the list of the references
+		// we do NOT needs to fetch (files inside the directory)
+		fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i"
+	)
+
+	// Start node
+	h := harness.NewT(t)
+	node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
+	client := node.GatewayClient()
+
+	// Import fixtures
+	r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car")
+	assert.Nil(t, err)
+	defer r.Close()
+	cid := node.IPFSDagImport(r)
+	assert.Equal(t, fixtureCid, cid)
+
+	t.Run("Fetch HAMT directory succeeds with minimal refs", func(t *testing.T) {
+		t.Parallel()
+		resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid))
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+	})
+
+	t.Run("Non-minimal refs are not present in the repository", func(t *testing.T) {
+		t.Parallel()
+
+		// Fetch list with refs of files that should NOT be available locally.
+		resp := client.Get(fmt.Sprintf("/ipfs/%s/files_refs", fixtureCid))
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		files := strings.Split(strings.TrimSpace(resp.Body), "\n")
+		assert.Len(t, files, 10100)
+
+		// Shuffle the files list and try fetching the first 200.
+		rand.Seed(time.Now().UnixNano())
+		rand.Shuffle(len(files), func(i, j int) { files[i], files[j] = files[j], files[i] })
+		for _, cid := range files[:200] {
+			resp = client.Get(fmt.Sprintf("/ipfs/%s", cid))
+			assert.Equal(t, http.StatusNotFound, resp.StatusCode)
+		}
+	})
+}
+
+func TestGatewayMultiRange(t *testing.T) {
+	t.Parallel()
+
+	const (
+		// fileCid is the CID of the large HAMT-sharded file.
+		fileCid = "bafybeibkzwf3ffl44yfej6ak44i7aly7rb4udhz5taskreec7qemmw5jiu"
+
+		// fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG
+		// representing the minimal set of blocks necessary for a simple byte range request.
+		fixtureCid = "bafybeihqs4hdx64a6wmrclp3a2pwxkd5prwdos45bdftpegls5ktzspi7a"
+	)
+
+	// Start node
+	h := harness.NewT(t)
+	node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
+	client := node.GatewayClient()
+
+	// Import fixtures
+	r, err := os.Open("./fixtures/TestGatewayMultiRange.car")
+	assert.Nil(t, err)
+	defer r.Close()
+	cid := node.IPFSDagImport(r)
+	assert.Equal(t, fixtureCid, cid)
+
+	t.Run("Succeeds to fetch range of blocks we have", func(t *testing.T) {
+		t.Parallel()
+
+		resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) {
+			r.Header.Set("Range", "bytes=2000-2002, 40000000000-40000000002")
+		})
+		assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
+		assert.Contains(t, resp.Body, "Content-Type: application/octet-stream")
+		assert.Contains(t, resp.Body, "Content-Range: bytes 2000-2002/87186935127")
+		assert.Contains(t, resp.Body, "Content-Range: bytes 40000000000-40000000002/87186935127")
+	})
+
+	t.Run("Fail to fetch range of blocks we do not have", func(t *testing.T) {
+		t.Parallel()
+
+		req, err := http.NewRequest(http.MethodGet, client.BuildURL(fmt.Sprintf("/ipfs/%s", fileCid)), nil)
+		assert.Nil(t, err)
+		req.Header.Set("Range", "bytes=1000-1100, 87186935125-87186935127")
+		httpResp, err := client.Client.Do(req)
+		assert.Nil(t, err)
+		assert.Equal(t, http.StatusPartialContent, httpResp.StatusCode)
+		_, err = io.ReadAll(httpResp.Body)
+		assert.Equal(t, err, io.ErrUnexpectedEOF)
+	})
+}

test/cli/harness/ipfs.go

@@ -78,3 +78,20 @@ func (n *Node) IPFSAdd(content io.Reader, args ...string) string {
 	log.Debugf("add result: %q", out)
 	return out
 }
+
+func (n *Node) IPFSDagImport(content io.Reader, args ...string) string {
+	log.Debugf("node %d dag import with args: %v", n.ID, args)
+	fullArgs := []string{"dag", "import"}
+	fullArgs = append(fullArgs, args...)
+	res := n.Runner.MustRun(RunRequest{
+		Path:    n.IPFSBin,
+		Args:    fullArgs,
+		CmdOpts: []CmdOpt{RunWithStdin(content)},
+	})
+	out := strings.TrimSpace(res.Stdout.String())
+	out = strings.TrimPrefix(out, "Pinned root")
+	out = strings.TrimSuffix(out, "success")
+	out = strings.TrimSpace(out)
+	log.Debugf("add result: %q", out)
+	return out
+}

test/sharness/t0115-gateway-dir-listing.sh

@@ -8,61 +8,21 @@ test_description="Test directory listing (dir-index-html) on the HTTP gateway"
 . lib/test-lib.sh
 
 ## ============================================================================
-## HAMT-sharded directory test. We must execute this test first as we want to
-## start from an empty repository and count the exact number of refs we want.
+## Start IPFS Node and prepare test CIDs
 ## ============================================================================
 
 test_expect_success "ipfs init" '
-  export IPFS_PATH="$(pwd)/.ipfs-for-hamt" &&
-  ipfs init --empty-repo --profile=test > /dev/null
+  export IPFS_PATH="$(pwd)/.ipfs" &&
+  ipfs init --profile=test > /dev/null
 '
 
 test_launch_ipfs_daemon_without_network
 
-# HAMT_CID is a hamt-sharded-directory that has 10k of items
-HAMT_CID=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm
-
-# HAMT_REFS_CID is root of a DAG that is a subset of HAMT_CID DAG
-# representing minimal set of block necessary for directory listing
-# without fetching ANY root blocks of child items.
-HAMT_REFS_CID=bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a
-HAMT_REFS_COUNT=963
-
-test_expect_success "hamt: import fixture with necessary refs" '
-  ipfs dag import ../t0115-gateway-dir-listing/hamt-refs.car &&
-  ipfs refs local | wc -l | tr -d " " > refs_count_actual &&
-  echo $HAMT_REFS_COUNT > refs_count_expected &&
-  test_cmp refs_count_expected refs_count_actual
-'
-
-# we want to confirm that code responsible for directory listing works
-# with minimal set of blocks, and does not fetch entire thing (regression test)
-test_expect_success "hamt: fetch directory from gateway in offline mode" '
-  curl --max-time 30 -sD - http://127.0.0.1:$GWAY_PORT/ipfs/$HAMT_CID/ > hamt_list_response &&
-  ipfs refs local | wc -l | tr -d " " > refs_count_actual &&
-  echo $HAMT_REFS_COUNT > refs_count_expected &&
-  test_cmp refs_count_expected refs_count_actual
-'
-
-test_kill_ipfs_daemon
-
-## ============================================================================
-## Start IPFS Node and prepare test CIDs
-## ============================================================================
-
-test_expect_success "ipfs init" '
-  export IPFS_PATH="$(pwd)/.ipfs" &&
-  ipfs init --empty-repo --profile=test > /dev/null
-'
-#
 # Import test case
 # See the static fixtures in ./t0115-gateway-dir-listing/
-test_expect_success "add remaining test fixtures" '
+test_expect_success "Add the test directory" '
   ipfs dag import ../t0115-gateway-dir-listing/fixtures.car
 '
-
-test_launch_ipfs_daemon_without_network
-
 DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir/
 FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
 FILE_SIZE=34

test/sharness/t0115-gateway-dir-listing/README.md

@@ -2,12 +2,8 @@
 
 - fixtures.car
   - raw CARv1
-- hamt-refs.car
-  - raw CARv1 containing all the necessary blocks to present a directory listing
-    for `bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm`, which is
-    a directory containing over 10k items.
 
-fixtures.car generated with:
+generated with:
 
 ```sh
 # using ipfs version 0.18.1
@@ -34,20 +30,3 @@ ipfs dag export ${DIR_CID} > ./fixtures.car
 # FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
 # FILE_SIZE=34
 ```
-
-hamt-refs.car generated with:
-
-```sh
-# using ipfs version 0.18.1
-export IPFS_PATH=$(mktemp -d)
-ipfs init --empty-repo
-ipfs daemon &
-curl http://127.0.0.1:8080/ipfs/bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm/
-killall ipfs
-ipfs refs local >> refs
-ipfs files mkdir --cid-version 1 /Test
-cat refs | xargs -I {} ipfs files cp /ipfs/{} /Test/{}
-ipfs files stat /Test
-# Grab CID: bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a
-ipfs dag export bafybeicv4utmj46mgbpamvw2k6zg4qjs5yvspfnlxz2y6iuey5smjfcn4a > ./refs.car
-```