diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go index 3d6966ac1ef..b41c7d0f233 100644 --- a/core/coreapi/unixfs.go +++ b/core/coreapi/unixfs.go @@ -271,6 +271,10 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se lnk.Type = coreiface.TFile lnk.Size = linkres.Link.Size case cid.DagProtobuf: + if settings.UseCumulativeSize { + lnk.Size = linkres.Link.Size + } + if !settings.ResolveChildren { break } @@ -296,7 +300,9 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se lnk.Type = coreiface.TSymlink lnk.Target = string(d.Data()) } - lnk.Size = d.FileSize() + if !settings.UseCumulativeSize { + lnk.Size = d.FileSize() + } } } diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go index 94474b4c222..5e90a8a7996 100644 --- a/core/corehttp/gateway_handler_unixfs_dir.go +++ b/core/corehttp/gateway_handler_unixfs_dir.go @@ -11,12 +11,9 @@ import ( "github.com/dustin/go-humanize" cid "github.com/ipfs/go-cid" files "github.com/ipfs/go-ipfs-files" - format "github.com/ipfs/go-ipld-format" - merkledag "github.com/ipfs/go-merkledag" path "github.com/ipfs/go-path" "github.com/ipfs/go-path/resolver" - "github.com/ipfs/go-unixfs" - "github.com/ipfs/go-unixfs/hamt" + options "github.com/ipfs/interface-go-ipfs-core/options" ipath "github.com/ipfs/interface-go-ipfs-core/path" "github.com/ipfs/kubo/assets" "github.com/ipfs/kubo/tracing" @@ -108,17 +105,26 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit return } - // Optimization: use Dag.Get to fetch the children links of this directory - // instead of UnixFS.LS. Dag.Get is faster and also provides a Size field - // that is good enough for a directory listing. - links, err := i.getUnixFsLinks(ctx, resolvedPath.Cid()) + // Optimization: use Unixfs.Ls without resolving children, but using the + // cumulative DAG size as the file size. This allows for a fast listing + // while keeping a good enough Size field. + results, err := i.api.Unixfs().Ls(ctx, + resolvedPath, + options.Unixfs.ResolveChildren(false), + options.Unixfs.UseCumulativeSize(true), + ) if err != nil { internalWebError(w, err) return } - dirListing := make([]directoryItem, 0, len(links)) - for _, link := range links { + dirListing := make([]directoryItem, 0, len(results)) + for link := range results { + if link.Err != nil { + internalWebError(w, err) + return + } + hash := link.Cid.String() di := directoryItem{ Size: humanize.Bytes(uint64(link.Size)), @@ -202,36 +208,3 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit func getDirListingEtag(dirCid cid.Cid) string { return `"DirIndex-` + assets.AssetHash + `_CID-` + dirCid.String() + `"` } - -func (i *gatewayHandler) getUnixFsLinks(ctx context.Context, cid cid.Cid) ([]*format.Link, error) { - obj, err := i.api.Dag().Get(ctx, cid) - if err != nil { - return nil, err - } - - protoNode, ok := obj.(*merkledag.ProtoNode) - if !ok { - return obj.Links(), nil - } - - fsNode, err := unixfs.FSNodeFromBytes(protoNode.Data()) - if err != nil { - return nil, err - } - - if fsNode.Type() == unixfs.THAMTShard { - shard, err := hamt.NewHamtFromDag(i.api.Dag(), obj) - if err != nil { - return nil, err - } - - links, err := shard.EnumLinks(ctx) - if err != nil { - return nil, err - } - - return links, nil - } else { - return obj.Links(), nil - } -} diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index 8547eb31f80..a53cadcbee3 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -8,7 +8,7 @@ replace github.com/ipfs/kubo => ./../../.. require ( github.com/ipfs/go-ipfs-files v0.2.0 - github.com/ipfs/interface-go-ipfs-core v0.7.0 + github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.23.4 github.com/multiformats/go-multiaddr v0.8.0 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 17252e076a0..f2611a6857f 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -624,8 +624,8 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= -github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js= -github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY= +github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e h1:paBQ+cqSEN5Vx4S4i4jiBZfpuHZwoLTC2JD+ZXijqeQ= +github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e/go.mod h1:X/udt0qeqxXlgv69JQ8g38gWy4LrCyOuav6f7KDoJMo= github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk= github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4= github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ= diff --git a/go.mod b/go.mod index eb8be2b589c..e891390e484 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/ipfs/go-unixfs v0.4.1 github.com/ipfs/go-unixfsnode v1.4.0 github.com/ipfs/go-verifcid v0.0.2 - github.com/ipfs/interface-go-ipfs-core v0.7.0 + github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e github.com/ipld/go-car v0.4.0 github.com/ipld/go-car/v2 v2.4.0 github.com/ipld/go-codec-dagpb v1.4.1 diff --git a/go.sum b/go.sum index 7a3c3e8ce89..ed989c81f58 100644 --- a/go.sum +++ b/go.sum @@ -650,8 +650,8 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= -github.com/ipfs/interface-go-ipfs-core v0.7.0 h1:7tb+2upz8oCcjIyjo1atdMk+P+u7wPmI+GksBlLE8js= -github.com/ipfs/interface-go-ipfs-core v0.7.0/go.mod h1:lF27E/nnSPbylPqKVXGZghal2hzifs3MmjyiEjnc9FY= +github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e h1:paBQ+cqSEN5Vx4S4i4jiBZfpuHZwoLTC2JD+ZXijqeQ= +github.com/ipfs/interface-go-ipfs-core v0.7.1-0.20221208145217-d5ff04b0800e/go.mod h1:X/udt0qeqxXlgv69JQ8g38gWy4LrCyOuav6f7KDoJMo= github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk= github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4= github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=