Merge pull request ipfs/kubo#9234 from ipfs/release-v0.15.0

chore: Release v0.15.0

This commit was moved from ipfs/kubo@3ae52a4

gateway/core/corehttp/gateway.go

@@ -1,18 +1,20 @@
 package corehttp
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/http"
 	"sort"
 
+	coreiface "github.com/ipfs/interface-go-ipfs-core"
+	options "github.com/ipfs/interface-go-ipfs-core/options"
+	path "github.com/ipfs/interface-go-ipfs-core/path"
 	version "github.com/ipfs/kubo"
 	core "github.com/ipfs/kubo/core"
 	coreapi "github.com/ipfs/kubo/core/coreapi"
-	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
-
-	options "github.com/ipfs/interface-go-ipfs-core/options"
 	id "github.com/libp2p/go-libp2p/p2p/protocol/identify"
+	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
 )
 
 type GatewayConfig struct {
@@ -22,6 +24,21 @@ type GatewayConfig struct {
 	FastDirIndexThreshold int
 }
 
+// NodeAPI defines the minimal set of API services required by a gateway handler
+type NodeAPI interface {
+	// Unixfs returns an implementation of Unixfs API
+	Unixfs() coreiface.UnixfsAPI
+
+	// Block returns an implementation of Block API
+	Block() coreiface.BlockAPI
+
+	// Dag returns an implementation of Dag API
+	Dag() coreiface.APIDagService
+
+	// ResolvePath resolves the path using Unixfs resolver
+	ResolvePath(context.Context, path.Path) (path.Resolved, error)
+}
+
 // A helper function to clean up a set of headers:
 // 1. Canonicalizes.
 // 2. Deduplicates.
@@ -59,49 +76,19 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 			headers[http.CanonicalHeaderKey(h)] = v
 		}
 
-		// Hard-coded headers.
-		const ACAHeadersName = "Access-Control-Allow-Headers"
-		const ACEHeadersName = "Access-Control-Expose-Headers"
-		const ACAOriginName = "Access-Control-Allow-Origin"
-		const ACAMethodsName = "Access-Control-Allow-Methods"
+		AddAccessControlHeaders(headers)
 
-		if _, ok := headers[ACAOriginName]; !ok {
-			// Default to *all*
-			headers[ACAOriginName] = []string{"*"}
-		}
-		if _, ok := headers[ACAMethodsName]; !ok {
-			// Default to GET
-			headers[ACAMethodsName] = []string{http.MethodGet}
+		offlineApi, err := api.WithOptions(options.Api.Offline(true))
+		if err != nil {
+			return nil, err
 		}
 
-		headers[ACAHeadersName] = cleanHeaderSet(
-			append([]string{
-				"Content-Type",
-				"User-Agent",
-				"Range",
-				"X-Requested-With",
-			}, headers[ACAHeadersName]...))
-
-		headers[ACEHeadersName] = cleanHeaderSet(
-			append([]string{
-				"Content-Length",
-				"Content-Range",
-				"X-Chunked-Output",
-				"X-Stream-Output",
-				"X-Ipfs-Path",
-				"X-Ipfs-Roots",
-			}, headers[ACEHeadersName]...))
-
-		var gateway http.Handler
-		gateway, err = newGatewayHandler(GatewayConfig{
+		gateway := NewGatewayHandler(GatewayConfig{
 			Headers:               headers,
 			Writable:              writable,
 			PathPrefixes:          cfg.Gateway.PathPrefixes,
 			FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
-		}, api)
-		if err != nil {
-			return nil, err
-		}
+		}, api, offlineApi)
 
 		gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
 
@@ -112,6 +99,50 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
 	}
 }
 
+// AddAccessControlHeaders adds default headers used for controlling
+// cross-origin requests. This function adds several values to the
+// Access-Control-Allow-Headers and Access-Control-Expose-Headers entries.
+// If the Access-Control-Allow-Origin entry is missing a value of '*' is
+// added, indicating that browsers should allow requesting code from any
+// origin to access the resource.
+// If the Access-Control-Allow-Methods entry is missing a value of 'GET' is
+// added, indicating that browsers may use the GET method when issuing cross
+// origin requests.
+func AddAccessControlHeaders(headers map[string][]string) {
+	// Hard-coded headers.
+	const ACAHeadersName = "Access-Control-Allow-Headers"
+	const ACEHeadersName = "Access-Control-Expose-Headers"
+	const ACAOriginName = "Access-Control-Allow-Origin"
+	const ACAMethodsName = "Access-Control-Allow-Methods"
+
+	if _, ok := headers[ACAOriginName]; !ok {
+		// Default to *all*
+		headers[ACAOriginName] = []string{"*"}
+	}
+	if _, ok := headers[ACAMethodsName]; !ok {
+		// Default to GET
+		headers[ACAMethodsName] = []string{http.MethodGet}
+	}
+
+	headers[ACAHeadersName] = cleanHeaderSet(
+		append([]string{
+			"Content-Type",
+			"User-Agent",
+			"Range",
+			"X-Requested-With",
+		}, headers[ACAHeadersName]...))
+
+	headers[ACEHeadersName] = cleanHeaderSet(
+		append([]string{
+			"Content-Length",
+			"Content-Range",
+			"X-Chunked-Output",
+			"X-Stream-Output",
+			"X-Ipfs-Path",
+			"X-Ipfs-Roots",
+		}, headers[ACEHeadersName]...))
+}
+
 func VersionOption() ServeOption {
 	return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
 		mux.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {

gateway/core/corehttp/gateway_handler.go

@@ -19,12 +19,12 @@ import (
 
 	cid "github.com/ipfs/go-cid"
 	files "github.com/ipfs/go-ipfs-files"
+	ipld "github.com/ipfs/go-ipld-format"
 	dag "github.com/ipfs/go-merkledag"
 	mfs "github.com/ipfs/go-mfs"
 	path "github.com/ipfs/go-path"
 	"github.com/ipfs/go-path/resolver"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
-	options "github.com/ipfs/interface-go-ipfs-core/options"
 	ipath "github.com/ipfs/interface-go-ipfs-core/path"
 	routing "github.com/libp2p/go-libp2p-core/routing"
 	prometheus "github.com/prometheus/client_golang/prometheus"
@@ -68,8 +68,8 @@ type redirectTemplateData struct {
 // (it serves requests like GET /ipfs/QmVRzPKPzNtSrEzBFm2UZfxmPAgnaLke4DMcerbsGGSaFe/link)
 type gatewayHandler struct {
 	config     GatewayConfig
-	api        coreiface.CoreAPI
-	offlineApi coreiface.CoreAPI
+	api        NodeAPI
+	offlineApi NodeAPI
 
 	// generic metrics
 	firstContentBlockGetMetric *prometheus.HistogramVec
@@ -213,11 +213,13 @@ func newGatewayHistogramMetric(name string, help string) *prometheus.HistogramVe
 	return histogramMetric
 }
 
-func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) (*gatewayHandler, error) {
-	offlineApi, err := api.WithOptions(options.Api.Offline(true))
-	if err != nil {
-		return nil, err
-	}
+// NewGatewayHandler returns an http.Handler that can act as a gateway to IPFS content
+// offlineApi is a version of the API that should not make network requests for missing data
+func NewGatewayHandler(c GatewayConfig, api NodeAPI, offlineApi NodeAPI) http.Handler {
+	return newGatewayHandler(c, api, offlineApi)
+}
+
+func newGatewayHandler(c GatewayConfig, api NodeAPI, offlineApi NodeAPI) *gatewayHandler {
 	i := &gatewayHandler{
 		config:     c,
 		api:        api,
@@ -262,7 +264,7 @@ func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) (*gatewayHandler,
 			"The time to receive the first UnixFS node on a GET from the gateway.",
 		),
 	}
-	return i, nil
+	return i
 }
 
 func parseIpfsPath(p string) (cid.Cid, string, error) {
@@ -389,8 +391,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
 			logger.Debugw("serve pretty 404 if present")
 			return
 		}
-
-		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
+		webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusBadRequest)
 		return
 	}
 
@@ -782,6 +783,8 @@ func webError(w http.ResponseWriter, message string, err error, defaultCode int)
 		webErrorWithCode(w, message, err, http.StatusNotFound)
 	} else if err == routing.ErrNotFound {
 		webErrorWithCode(w, message, err, http.StatusNotFound)
+	} else if ipld.IsNotFound(err) {
+		webErrorWithCode(w, message, err, http.StatusNotFound)
 	} else if err == context.DeadlineExceeded {
 		webErrorWithCode(w, message, err, http.StatusRequestTimeout)
 	} else {

gateway/core/corehttp/gateway_handler_unixfs.go

@@ -22,7 +22,7 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
 	// Handling UnixFS
 	dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
 	if err != nil {
-		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
+		webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusBadRequest)
 		return
 	}
 	defer dr.Close()

gateway/core/corehttp/gateway_handler_unixfs_dir.go

@@ -41,28 +41,30 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 	}
 	originalUrlPath := requestURI.Path
 
-	// Check if directory has index.html, if so, serveFile
-	idxPath := ipath.Join(contentPath, "index.html")
-	idx, err := i.api.Unixfs().Get(ctx, idxPath)
-	switch err.(type) {
-	case nil:
-		cpath := contentPath.String()
-		dirwithoutslash := cpath[len(cpath)-1] != '/'
+	// Ensure directory paths end with '/'
+	if originalUrlPath[len(originalUrlPath)-1] != '/' {
+		// don't redirect to trailing slash if it's go get
+		// https://github.com/ipfs/kubo/pull/3963
 		goget := r.URL.Query().Get("go-get") == "1"
-		if dirwithoutslash && !goget {
-			// See comment above where originalUrlPath is declared.
+		if !goget {
 			suffix := "/"
+			// preserve query parameters
 			if r.URL.RawQuery != "" {
-				// preserve query parameters
 				suffix = suffix + "?" + r.URL.RawQuery
 			}
-
+			// /ipfs/cid/foo?bar must be redirected to /ipfs/cid/foo/?bar
 			redirectURL := originalUrlPath + suffix
-			logger.Debugw("serving index.html file", "to", redirectURL, "status", http.StatusFound, "path", idxPath)
-			http.Redirect(w, r, redirectURL, http.StatusFound)
+			logger.Debugw("directory location moved permanently", "status", http.StatusMovedPermanently)
+			http.Redirect(w, r, redirectURL, http.StatusMovedPermanently)
 			return
 		}
+	}
 
+	// Check if directory has index.html, if so, serveFile
+	idxPath := ipath.Join(contentPath, "index.html")
+	idx, err := i.api.Unixfs().Get(ctx, idxPath)
+	switch err.(type) {
+	case nil:
 		f, ok := idx.(files.File)
 		if !ok {
 			internalWebError(w, files.ErrNotReader)
@@ -152,16 +154,18 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
 	// don't go further up than /ipfs/$hash/
 	pathSplit := path.SplitList(contentPath.String())
 	switch {
-	// keep backlink
+	// skip backlink when listing a content root
 	case len(pathSplit) == 3: // url: /ipfs/$hash
+		backLink = ""
 
-	// keep backlink
+	// skip backlink when listing a content root
 	case len(pathSplit) == 4 && pathSplit[3] == "": // url: /ipfs/$hash/
+		backLink = ""
 
 	// add the correct link depending on whether the path ends with a slash
 	default:
 		if strings.HasSuffix(backLink, "/") {
-			backLink += "./.."
+			backLink += ".."
 		} else {
 			backLink += "/.."
 		}

gateway/core/corehttp/gateway_test.go

@@ -235,17 +235,17 @@ func TestGatewayGet(t *testing.T) {
 		{"127.0.0.1:8080", "/", http.StatusNotFound, "404 page not found\n"},
 		{"127.0.0.1:8080", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"},
 		{"127.0.0.1:8080", k.String(), http.StatusOK, "fnord"},
-		{"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusNotFound, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"},
-		{"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/\\r\\n\\r\\nhello: " + namesys.ErrResolveFailed.Error() + "\n"},
+		{"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusBadRequest, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"},
+		{"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusBadRequest, "ipfs resolve -r /ipns/\\r\\n\\r\\nhello: " + namesys.ErrResolveFailed.Error() + "\n"},
 		{"127.0.0.1:8080", "/ipns/example.com", http.StatusOK, "fnord"},
 		{"example.com", "/", http.StatusOK, "fnord"},
 
 		{"working.example.com", "/", http.StatusOK, "fnord"},
 		{"double.example.com", "/", http.StatusOK, "fnord"},
 		{"triple.example.com", "/", http.StatusOK, "fnord"},
 		{"working.example.com", k.String(), http.StatusNotFound, "ipfs resolve -r /ipns/working.example.com" + k.String() + ": no link named \"ipfs\" under " + k.Cid().String() + "\n"},
-		{"broken.example.com", "/", http.StatusNotFound, "ipfs resolve -r /ipns/broken.example.com/: " + namesys.ErrResolveFailed.Error() + "\n"},
-		{"broken.example.com", k.String(), http.StatusNotFound, "ipfs resolve -r /ipns/broken.example.com" + k.String() + ": " + namesys.ErrResolveFailed.Error() + "\n"},
+		{"broken.example.com", "/", http.StatusBadRequest, "ipfs resolve -r /ipns/broken.example.com/: " + namesys.ErrResolveFailed.Error() + "\n"},
+		{"broken.example.com", k.String(), http.StatusBadRequest, "ipfs resolve -r /ipns/broken.example.com" + k.String() + ": " + namesys.ErrResolveFailed.Error() + "\n"},
 		// This test case ensures we don't treat the TLD as a file extension.
 		{"example.man", "/", http.StatusOK, "fnord"},
 	} {
@@ -380,9 +380,9 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// expect 302 redirect to same path, but with trailing slash
-	if res.StatusCode != 302 {
-		t.Errorf("status is %d, expected 302", res.StatusCode)
+	// expect 301 redirect to same path, but with trailing slash
+	if res.StatusCode != 301 {
+		t.Errorf("status is %d, expected 301", res.StatusCode)
 	}
 	hdr := res.Header["Location"]
 	if len(hdr) < 1 {
@@ -403,9 +403,9 @@ func TestIPNSHostnameRedirect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// expect 302 redirect to same path, but with prefix and trailing slash
-	if res.StatusCode != 302 {
-		t.Errorf("status is %d, expected 302", res.StatusCode)
+	// expect 301 redirect to same path, but with prefix and trailing slash
+	if res.StatusCode != 301 {
+		t.Errorf("status is %d, expected 301", res.StatusCode)
 	}
 	hdr = res.Header["Location"]
 	if len(hdr) < 1 {
@@ -492,7 +492,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"//example.net/\">example.net</a>/<a href=\"//example.net/foo%3F%20%23%3C%27\">foo? #&lt;&#39;</a>") {
 		t.Fatalf("expected a path in directory listing")
 	}
-	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/./..\">") {
+	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/..\">") {
 		t.Fatalf("expected backlink in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/file.txt\">") {
@@ -529,8 +529,8 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !matchPathOrBreadcrumbs(s, "/") {
 		t.Fatalf("expected a path in directory listing")
 	}
-	if !strings.Contains(s, "<a href=\"/\">") {
-		t.Fatalf("expected backlink in directory listing")
+	if strings.Contains(s, "<a href=\"/\">") {
+		t.Fatalf("expected no backlink in directory listing of the root CID")
 	}
 	if !strings.Contains(s, "<a href=\"/file.txt\">") {
 		t.Fatalf("expected file in directory listing")
@@ -566,7 +566,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
 	if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"//example.net/\">example.net</a>/<a href=\"//example.net/foo%3F%20%23%3C%27\">foo? #&lt;&#39;</a>/<a href=\"//example.net/foo%3F%20%23%3C%27/bar\">bar</a>") {
 		t.Fatalf("expected a path in directory listing")
 	}
-	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/bar/./..\">") {
+	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/bar/..\">") {
 		t.Fatalf("expected backlink in directory listing")
 	}
 	if !strings.Contains(s, "<a href=\"/foo%3F%20%23%3C%27/bar/file.txt\">") {

gateway/core/corehttp/hostname.go

@@ -324,7 +324,7 @@ func isKnownHostname(hostname string, knownGateways gatewayHosts) (gw *config.Ga
 func knownSubdomainDetails(hostname string, knownGateways gatewayHosts) (gw *config.GatewaySpec, gwHostname, ns, rootID string, ok bool) {
 	labels := strings.Split(hostname, ".")
 	// Look for FQDN of a known gateway hostname.
-	// Example: given "dist.ipfs.io.ipns.dweb.link":
+	// Example: given "dist.ipfs.tech.ipns.dweb.link":
 	// 1. Lookup "link" TLD in knownGateways: negative
 	// 2. Lookup "dweb.link" in knownGateways: positive
 	//