diff --git a/assets/dir-index-html/dir-index.html b/assets/dir-index-html/dir-index.html
index 1d00e5fe73f..d861cb65700 100644
--- a/assets/dir-index-html/dir-index.html
+++ b/assets/dir-index-html/dir-index.html
@@ -56,7 +56,7 @@
{{ if .Size }}
- {{ .Size }}
+ {{ .Size }}
{{ end }}
@@ -89,7 +89,7 @@
{{ end }}
- {{ .Size }} |
+ {{ .Size }} |
{{ end }}
diff --git a/assets/dir-index-html/src/dir-index.html b/assets/dir-index-html/src/dir-index.html
index a763b3e76bb..109c7afbf44 100644
--- a/assets/dir-index-html/src/dir-index.html
+++ b/assets/dir-index-html/src/dir-index.html
@@ -55,7 +55,7 @@
{{ if .Size }}
- {{ .Size }}
+ {{ .Size }}
{{ end }}
@@ -88,7 +88,7 @@
{{ end }}
- {{ .Size }} |
+ {{ .Size }} |
{{ end }}
diff --git a/assets/dir-index-html/test/main.go b/assets/dir-index-html/test/main.go
index 43b4a098101..c02523a9f40 100644
--- a/assets/dir-index-html/test/main.go
+++ b/assets/dir-index-html/test/main.go
@@ -12,15 +12,14 @@ const templateFile = "../dir-index.html"
// Copied from go-ipfs/core/corehttp/gateway_indexPage.go
type listingTemplateData struct {
- GatewayURL string
- DNSLink bool
- Listing []directoryItem
- Size string
- Path string
- Breadcrumbs []breadcrumb
- BackLink string
- Hash string
- FastDirIndexThreshold int
+ GatewayURL string
+ DNSLink bool
+ Listing []directoryItem
+ Size string
+ Path string
+ Breadcrumbs []breadcrumb
+ BackLink string
+ Hash string
}
type directoryItem struct {
diff --git a/config/gateway.go b/config/gateway.go
index 8b8c65d1db5..ad01b263b36 100644
--- a/config/gateway.go
+++ b/config/gateway.go
@@ -45,14 +45,6 @@ type Gateway struct {
// PathPrefixes was removed: https://github.com/ipfs/go-ipfs/issues/7702
PathPrefixes []string
- // FastDirIndexThreshold is the maximum number of items in a directory
- // before the Gateway switches to a shallow, faster listing which only
- // requires the root node. This allows for listing big directories fast,
- // without the linear slowdown caused by reading size metadata from child
- // nodes.
- // Setting to 0 will enable fast listings for all directories.
- FastDirIndexThreshold *OptionalInteger `json:",omitempty"`
-
// FIXME: Not yet implemented: https://github.com/ipfs/kubo/issues/8059
APICommands []string
diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go
index 3d6966ac1ef..753a3d3a0ef 100644
--- a/core/coreapi/unixfs.go
+++ b/core/coreapi/unixfs.go
@@ -271,32 +271,36 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
lnk.Type = coreiface.TFile
lnk.Size = linkres.Link.Size
case cid.DagProtobuf:
- if !settings.ResolveChildren {
- break
- }
-
- linkNode, err := linkres.Link.GetNode(ctx, api.dag)
- if err != nil {
- lnk.Err = err
- break
- }
-
- if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
- d, err := ft.FSNodeFromBytes(pn.Data())
+ if settings.ResolveChildren {
+ linkNode, err := linkres.Link.GetNode(ctx, api.dag)
if err != nil {
lnk.Err = err
break
}
- switch d.Type() {
- case ft.TFile, ft.TRaw:
- lnk.Type = coreiface.TFile
- case ft.THAMTShard, ft.TDirectory, ft.TMetadata:
- lnk.Type = coreiface.TDirectory
- case ft.TSymlink:
- lnk.Type = coreiface.TSymlink
- lnk.Target = string(d.Data())
+
+ if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
+ d, err := ft.FSNodeFromBytes(pn.Data())
+ if err != nil {
+ lnk.Err = err
+ break
+ }
+ switch d.Type() {
+ case ft.TFile, ft.TRaw:
+ lnk.Type = coreiface.TFile
+ case ft.THAMTShard, ft.TDirectory, ft.TMetadata:
+ lnk.Type = coreiface.TDirectory
+ case ft.TSymlink:
+ lnk.Type = coreiface.TSymlink
+ lnk.Target = string(d.Data())
+ }
+ if !settings.UseCumulativeSize {
+ lnk.Size = d.FileSize()
+ }
}
- lnk.Size = d.FileSize()
+ }
+
+ if settings.UseCumulativeSize {
+ lnk.Size = linkres.Link.Size
}
}
diff --git a/core/corehttp/gateway.go b/core/corehttp/gateway.go
index 0d0a234d946..334000b5ab3 100644
--- a/core/corehttp/gateway.go
+++ b/core/corehttp/gateway.go
@@ -18,9 +18,8 @@ import (
)
type GatewayConfig struct {
- Headers map[string][]string
- Writable bool
- FastDirIndexThreshold int
+ Headers map[string][]string
+ Writable bool
}
// NodeAPI defines the minimal set of API services required by a gateway handler
@@ -83,9 +82,8 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
}
gateway := NewGatewayHandler(GatewayConfig{
- Headers: headers,
- Writable: writable,
- FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
+ Headers: headers,
+ Writable: writable,
}, api, offlineAPI)
gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
diff --git a/core/corehttp/gateway_handler_unixfs_dir.go b/core/corehttp/gateway_handler_unixfs_dir.go
index 1c803b13b34..5e90a8a7996 100644
--- a/core/corehttp/gateway_handler_unixfs_dir.go
+++ b/core/corehttp/gateway_handler_unixfs_dir.go
@@ -105,25 +105,29 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
return
}
- // Optimization 1:
- // List children without fetching their root blocks (fast, but no size info)
- results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
+ // Optimization: use Unixfs.Ls without resolving children, but using the
+ // cumulative DAG size as the file size. This allows for a fast listing
+ // while keeping a good enough Size field.
+ results, err := i.api.Unixfs().Ls(ctx,
+ resolvedPath,
+ options.Unixfs.ResolveChildren(false),
+ options.Unixfs.UseCumulativeSize(true),
+ )
if err != nil {
internalWebError(w, err)
return
}
- // storage for directory listing
dirListing := make([]directoryItem, 0, len(results))
-
for link := range results {
if link.Err != nil {
internalWebError(w, err)
return
}
+
hash := link.Cid.String()
di := directoryItem{
- Size: "", // no size because we did not fetch child nodes
+ Size: humanize.Bytes(uint64(link.Size)),
Name: link.Name,
Path: gopath.Join(originalURLPath, link.Name),
Hash: hash,
@@ -132,21 +136,6 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
dirListing = append(dirListing, di)
}
- // Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold
- if len(dirListing) < i.config.FastDirIndexThreshold {
- dirit := dir.Entries()
- linkNo := 0
- for dirit.Next() {
- size := "?"
- if s, err := dirit.Node().Size(); err == nil {
- // Size may not be defined/supported. Continue anyways.
- size = humanize.Bytes(uint64(s))
- }
- dirListing[linkNo].Size = size
- linkNo++
- }
- }
-
// construct the correct back link
// https://github.com/ipfs/kubo/issues/1365
backLink := originalURLPath
@@ -195,15 +184,14 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
// See comment above where originalUrlPath is declared.
tplData := listingTemplateData{
- GatewayURL: gwURL,
- DNSLink: dnslink,
- Listing: dirListing,
- Size: size,
- Path: contentPath.String(),
- Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
- BackLink: backLink,
- Hash: hash,
- FastDirIndexThreshold: i.config.FastDirIndexThreshold,
+ GatewayURL: gwURL,
+ DNSLink: dnslink,
+ Listing: dirListing,
+ Size: size,
+ Path: contentPath.String(),
+ Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
+ BackLink: backLink,
+ Hash: hash,
}
logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
diff --git a/core/corehttp/gateway_indexPage.go b/core/corehttp/gateway_indexPage.go
index 19e444da3f0..b0db8ac1a18 100644
--- a/core/corehttp/gateway_indexPage.go
+++ b/core/corehttp/gateway_indexPage.go
@@ -12,15 +12,14 @@ import (
// structs for directory listing
type listingTemplateData struct {
- GatewayURL string
- DNSLink bool
- Listing []directoryItem
- Size string
- Path string
- Breadcrumbs []breadcrumb
- BackLink string
- Hash string
- FastDirIndexThreshold int
+ GatewayURL string
+ DNSLink bool
+ Listing []directoryItem
+ Size string
+ Path string
+ Breadcrumbs []breadcrumb
+ BackLink string
+ Hash string
}
type directoryItem struct {
diff --git a/docs/changelogs/v0.18.md b/docs/changelogs/v0.18.md
index 2bd6dc192f5..4001f606f70 100644
--- a/docs/changelogs/v0.18.md
+++ b/docs/changelogs/v0.18.md
@@ -11,6 +11,7 @@ Below is an outline of all that is in this release, so you get a sense of all th
- [Overview](#overview)
- [🔦 Highlights](#-highlights)
- [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways)
+ - [🐎 Fast directory listings with DAG sizes](#-fast-directory-listings-with-dag-sizes)
- [Content Routing](#content-routing)
- [Provider Record Republish and Expiration](#provider-record-republish-and-expiration)
- [Lowered `ConnMgr`](#lowered-connmgr)
@@ -71,6 +72,16 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq
}
```
+#### 🐎 Fast directory listings with DAG sizes
+
+Fast listings are now enabled for _all_ UnixFS directories: big and small.
+There is no linear slowdown caused by reading size metadata from child nodes,
+and the size of DAG representing child items is always present.
+
+As an example, the CID
+`bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm` represents UnixFS
+directory with over 10k (10100) of files. Listing big directories was fast
+since Kubo 0.13, but in this release it will also include the size column.
#### Content Routing
diff --git a/docs/config.md b/docs/config.md
index 0c42296bf16..452dc1cace6 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -672,17 +672,7 @@ Type: `string` (url)
### `Gateway.FastDirIndexThreshold`
-The maximum number of items in a directory before the Gateway switches
-to a shallow, faster listing which only requires the root node.
-
-This allows for fast listings of big directories, without the linear slowdown caused
-by reading size metadata from child nodes.
-
-Setting to 0 will enable fast listings for all directories.
-
-Default: `100`
-
-Type: `optionalInteger`
+**REMOVED**: this option is [no longer necessary](https://github.com/ipfs/kubo/pull/9481). Ignored since [Kubo 0.18](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.18.md).
### `Gateway.Writable`
diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
index 031f7ef6806..1c40cfe81ea 100644
--- a/docs/examples/kubo-as-a-library/go.mod
+++ b/docs/examples/kubo-as-a-library/go.mod
@@ -8,7 +8,7 @@ replace github.com/ipfs/kubo => ./../../..
require (
github.com/ipfs/go-ipfs-files v0.2.0
- github.com/ipfs/interface-go-ipfs-core v0.8.0
+ github.com/ipfs/interface-go-ipfs-core v0.8.1
github.com/ipfs/kubo v0.14.0-rc1
github.com/libp2p/go-libp2p v0.24.1
github.com/multiformats/go-multiaddr v0.8.0
diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
index 099e001597b..f2d9d7ebdde 100644
--- a/docs/examples/kubo-as-a-library/go.sum
+++ b/docs/examples/kubo-as-a-library/go.sum
@@ -598,8 +598,8 @@ github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuF
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
-github.com/ipfs/interface-go-ipfs-core v0.8.0 h1:pNs34l947fvNOh+XEjXnHW/GV6HXmEzJNeqZFhX4GoQ=
-github.com/ipfs/interface-go-ipfs-core v0.8.0/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
+github.com/ipfs/interface-go-ipfs-core v0.8.1 h1:nuFG0YJ429Wd5gtRb3ivlblpknZ5VfDVKZkmOG2TnNQ=
+github.com/ipfs/interface-go-ipfs-core v0.8.1/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=
diff --git a/go.mod b/go.mod
index a1472360405..ff21a3cd95b 100644
--- a/go.mod
+++ b/go.mod
@@ -63,7 +63,7 @@ require (
github.com/ipfs/go-unixfs v0.4.1
github.com/ipfs/go-unixfsnode v1.4.0
github.com/ipfs/go-verifcid v0.0.2
- github.com/ipfs/interface-go-ipfs-core v0.8.0
+ github.com/ipfs/interface-go-ipfs-core v0.8.1
github.com/ipld/go-car v0.4.0
github.com/ipld/go-car/v2 v2.4.0
github.com/ipld/go-codec-dagpb v1.4.1
diff --git a/go.sum b/go.sum
index 524b69951d1..f890f22b08a 100644
--- a/go.sum
+++ b/go.sum
@@ -625,8 +625,8 @@ github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuF
github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
-github.com/ipfs/interface-go-ipfs-core v0.8.0 h1:pNs34l947fvNOh+XEjXnHW/GV6HXmEzJNeqZFhX4GoQ=
-github.com/ipfs/interface-go-ipfs-core v0.8.0/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
+github.com/ipfs/interface-go-ipfs-core v0.8.1 h1:nuFG0YJ429Wd5gtRb3ivlblpknZ5VfDVKZkmOG2TnNQ=
+github.com/ipfs/interface-go-ipfs-core v0.8.1/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=
diff --git a/test/sharness/t0115-gateway-dir-listing.sh b/test/sharness/t0115-gateway-dir-listing.sh
index 4b8cf7bc248..708e0c4cf8b 100755
--- a/test/sharness/t0115-gateway-dir-listing.sh
+++ b/test/sharness/t0115-gateway-dir-listing.sh
@@ -163,28 +163,6 @@ test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.te
test_should_contain "" list_response
'
-## ============================================================================
-## Test dir listing of a big directory
-## ============================================================================
-
-test_expect_success "dir listing should resolve child sizes if under Gateway.FastDirIndexThreshold" '
- curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
- test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
- test_should_contain ">${FILE_SIZE} B" list_response
-'
-
-# force fast dir index for all responses
-ipfs config --json Gateway.FastDirIndexThreshold 0
-# restart daemon to apply config changes
-test_kill_ipfs_daemon
-test_launch_ipfs_daemon
-
-test_expect_success "dir listing should not resolve child sizes beyond Gateway.FastDirIndexThreshold" '
- curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
- test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
- test_should_not_contain ">${FILE_SIZE} B" list_response
-'
-
## ============================================================================
## End of tests, cleanup
## ============================================================================