refactor: add UseCumulativeSize to UnixfsLs instead

core/coreapi/unixfs.go

@@ -271,6 +271,10 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
 		lnk.Type = coreiface.TFile
 		lnk.Size = linkres.Link.Size
 	case cid.DagProtobuf:
+		if settings.UseCumulativeSize {
+			lnk.Size = linkres.Link.Size
+		}
+
 		if !settings.ResolveChildren {
 			break
 		}
@@ -296,7 +300,9 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
 				lnk.Type = coreiface.TSymlink
 				lnk.Target = string(d.Data())
 			}
-			lnk.Size = d.FileSize()
+			if !settings.UseCumulativeSize {
+				lnk.Size = d.FileSize()
+			}
 		}
 	}
 

core/corehttp/gateway_handler_unixfs_dir.go

@@ -11,12 +11,9 @@ import (
 	"github.com/dustin/go-humanize"
 	cid "github.com/ipfs/go-cid"
 	files "github.com/ipfs/go-ipfs-files"
-	format "github.com/ipfs/go-ipld-format"
-	merkledag "github.com/ipfs/go-merkledag"
 	path "github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
-	"github.com/ipfs/go-unixfs"
-	"github.com/ipfs/go-unixfs/hamt"
+	options "github.com/ipfs/interface-go-ipfs-core/options"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	"github.com/ipfs/kubo/assets"
 	"github.com/ipfs/kubo/tracing"
@@ -108,17 +105,26 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 		return
 	}
 
-	// Optimization: use Dag.Get to fetch the children links of this directory
-	// instead of UnixFS.LS. Dag.Get is faster and also provides a Size field
-	// that is good enough for a directory listing.
-	links, err := i.getUnixFsLinks(ctx, resolvedPath.Cid())
+	// Optimization: use Unixfs.Ls without resolving children, but using the
+	// cumulative DAG size as the file size. This allows for a fast listing
+	// while keeping a good enough Size field.
+	results, err := i.api.Unixfs().Ls(ctx,
+		resolvedPath,
+		options.Unixfs.ResolveChildren(false),
+		options.Unixfs.UseCumulativeSize(true),
+	)
 	if err != nil {
 		internalWebError(w, err)
 		return
 	}
 
-	dirListing := make([]directoryItem, 0, len(links))
-	for _, link := range links {
+	dirListing := make([]directoryItem, 0, len(results))
+	for link := range results {
+		if link.Err != nil {
+			internalWebError(w, err)
+			return
+		}
+
 		hash := link.Cid.String()
 		di := directoryItem{
 			Size:      humanize.Bytes(uint64(link.Size)),
@@ -202,36 +208,3 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 func getDirListingEtag(dirCid cid.Cid) string {
 	return `"DirIndex-` + assets.AssetHash + `_CID-` + dirCid.String() + `"`
 }
-
-func (i *gatewayHandler) getUnixFsLinks(ctx context.Context, cid cid.Cid) ([]*format.Link, error) {
-	obj, err := i.api.Dag().Get(ctx, cid)
-	if err != nil {
-		return nil, err
-	}
-
-	protoNode, ok := obj.(*merkledag.ProtoNode)
-	if !ok {
-		return obj.Links(), nil
-	}
-
-	fsNode, err := unixfs.FSNodeFromBytes(protoNode.Data())
-	if err != nil {
-		return nil, err
-	}
-
-	if fsNode.Type() == unixfs.THAMTShard {
-		shard, err := hamt.NewHamtFromDag(i.api.Dag(), obj)
-		if err != nil {
-			return nil, err
-		}
-
-		links, err := shard.EnumLinks(ctx)
-		if err != nil {
-			return nil, err
-		}
-
-		return links, nil
-	} else {
-		return obj.Links(), nil
-	}
-}

docs/examples/kubo-as-a-library/go.mod

@@ -8,7 +8,7 @@ replace github.com/ipfs/kubo => ./../../..
 
 require (
 	github.com/ipfs/go-ipfs-files v0.2.0
-	github.com/ipfs/interface-go-ipfs-core v0.7.0
+	github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.23.4
 	github.com/multiformats/go-multiaddr v0.8.0

docs/examples/kubo-as-a-library/go.sum

@@ -624,8 +624,8 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ
 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
 github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
 github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
-github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js=
-github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY=
+github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e h1:paBQ+cqSEN5Vx4S4i4jiBZfpuHZwoLTC2JD+ZXijqeQ=
+github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e/go.mod h1:X/udt0qeqxXlgv69JQ8g38gWy4LrCyOuav6f7KDoJMo=
 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
 github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=

go.mod

@@ -62,7 +62,7 @@ require (
 	github.com/ipfs/go-unixfs v0.4.1
 	github.com/ipfs/go-unixfsnode v1.4.0
 	github.com/ipfs/go-verifcid v0.0.2
-	github.com/ipfs/interface-go-ipfs-core v0.7.0
+	github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e
 	github.com/ipld/go-car v0.4.0
 	github.com/ipld/go-car/v2 v2.4.0
 	github.com/ipld/go-codec-dagpb v1.4.1

go.sum

@@ -650,8 +650,8 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ
 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
 github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
 github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o=
-github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js=
-github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY=
+github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e h1:paBQ+cqSEN5Vx4S4i4jiBZfpuHZwoLTC2JD+ZXijqeQ=
+github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e/go.mod h1:X/udt0qeqxXlgv69JQ8g38gWy4LrCyOuav6f7KDoJMo=
 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
 github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=