From 842ff8e9074e91c55a765a72c4e16ae49743cb68 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 25 May 2020 16:15:34 +0200 Subject: [PATCH] wip: support long CIDs in subdomains with TLS This is WIP Goal: add subdomain gateway support for CIDs longer than 63 characters in a way that fits in a single DNS label to enable TLS for every root. License: MIT Signed-off-by: Marcin Rataj --- core/corehttp/hostname.go | 105 +++++++++++++++++----- core/corehttp/hostname_test.go | 54 ++++++++--- test/sharness/t0114-gateway-subdomains.sh | 39 +++++++- 3 files changed, 164 insertions(+), 34 deletions(-) diff --git a/core/corehttp/hostname.go b/core/corehttp/hostname.go index d8656da23ec..5f251926378 100644 --- a/core/corehttp/hostname.go +++ b/core/corehttp/hostname.go @@ -41,12 +41,15 @@ var defaultKnownGateways = map[string]config.GatewaySpec{ "dweb.link": subdomainGatewaySpec, } +// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7) +const dnsLabelMaxLength int = 63 + // HostnameOption rewrites an incoming request based on the Host header. func HostnameOption() ServeOption { return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) { childMux := http.NewServeMux() - coreApi, err := coreapi.NewCoreAPI(n) + coreAPI, err := coreapi.NewCoreAPI(n) if err != nil { return nil, err } @@ -94,7 +97,12 @@ func HostnameOption() ServeOption { if gw.UseSubdomains { // Yes, redirect if applicable // Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link - if newURL, ok := toSubdomainURL(r.Host, r.URL.Path, r); ok { + newURL, err := toSubdomainURL(r.Host, r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { // Just to be sure single Origin can't be abused in // web browsers that ignored the redirect for some // reason, Clear-Site-Data header clears browsing @@ -124,7 +132,7 @@ func HostnameOption() ServeOption { // Not a whitelisted path // Try DNSLink, if it was not explicitly disabled for the hostname - if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) { + if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -151,16 +159,44 @@ func HostnameOption() ServeOption { return } - // Do we need to fix multicodec in PeerID represented as CIDv1? - if isPeerIDNamespace(ns) { - keyCid, err := cid.Decode(rootID) - if err == nil && keyCid.Type() != cid.Libp2pKey { - if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok { - // Redirect to CID fixed inside of toSubdomainURL() + // Check if rootID is a valid CID + if rootCID, err := cid.Decode(rootID); err == nil { + // Do we need to redirect root CID to a canonical DNS representation? + dnsCID, err := toDNSPrefix(rootID, rootCID) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if !strings.HasPrefix(r.Host, dnsCID) { + dnsPrefix := "/" + ns + "/" + dnsCID + newURL, err := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to deterministic CID to ensure CID + // always gets the same Origin on the web http.Redirect(w, r, newURL, http.StatusMovedPermanently) return } } + + // Do we need to fix multicodec in PeerID represented as CIDv1? + if isPeerIDNamespace(ns) { + if rootCID.Type() != cid.Libp2pKey { + newURL, err := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if newURL != "" { + // Redirect to CID fixed inside of toSubdomainURL() + http.Redirect(w, r, newURL, http.StatusMovedPermanently) + return + } + } + } } // Rewrite the path to not use subdomains @@ -176,7 +212,7 @@ func HostnameOption() ServeOption { // 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)? // 2. does Host header include a fully qualified domain name (FQDN)? // 3. does DNSLink record exist in DNS? - if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) { + if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path childMux.ServeHTTP(w, r) @@ -266,18 +302,38 @@ func isPeerIDNamespace(ns string) bool { } } +// Converts an identifier to DNS-safe representation that fits in 63 characters +func toDNSPrefix(rootID string, rootCID cid.Cid) (prefix string, err error) { + // Return as-is if things fit + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Convert to Base36 and see if that helped + rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36) + if err != nil { + return "", err + } + if len(rootID) <= dnsLabelMaxLength { + return rootID, nil + } + + // Can't win with DNS at this point, return error + return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID) +} + // Converts a hostname/path to a subdomain-based URL, if applicable. -func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) { +func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, err error) { var scheme, ns, rootID, rest string query := r.URL.RawQuery parts := strings.SplitN(path, "/", 4) - safeRedirectURL := func(in string) (out string, ok bool) { + safeRedirectURL := func(in string) (out string, err error) { safeURI, err := url.ParseRequestURI(in) if err != nil { - return "", false + return "", err } - return safeURI.String(), true + return safeURI.String(), nil } // Support X-Forwarded-Proto if added by a reverse proxy @@ -297,11 +353,11 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok ns = parts[1] rootID = parts[2] default: - return "", false + return "", nil } if !isSubdomainNamespace(ns) { - return "", false + return "", nil } // add prefix if query is present @@ -320,8 +376,8 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok } // If rootID is a CID, ensure it uses DNS-friendly text representation - if rootCid, err := cid.Decode(rootID); err == nil { - multicodec := rootCid.Type() + if rootCID, err := cid.Decode(rootID); err == nil { + multicodec := rootCID.Type() // PeerIDs represented as CIDv1 are expected to have libp2p-key // multicodec (https://github.com/libp2p/specs/pull/209). @@ -334,11 +390,16 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok // if object turns out to be a valid CID, // ensure text representation used in subdomain is CIDv1 in Base32 // https://github.com/ipfs/in-web-browsers/issues/89 - rootID, err = cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base32) + rootCID = cid.NewCidV1(multicodec, rootCID.Hash()) + rootID, err = rootCID.StringOfBase(mbase.Base32) + if err != nil { + return "", err + } + + // make sure CID fits in DNS label + rootID, err = toDNSPrefix(rootID, rootCID) if err != nil { - // should not error, but if it does, its clealy not possible to - // produce a subdomain URL - return "", false + return "", err } } diff --git a/core/corehttp/hostname_test.go b/core/corehttp/hostname_test.go index 9a297464891..e6e17f83a55 100644 --- a/core/corehttp/hostname_test.go +++ b/core/corehttp/hostname_test.go @@ -1,9 +1,11 @@ package corehttp import ( + "errors" "net/http/httptest" "testing" + cid "github.com/ipfs/go-cid" config "github.com/ipfs/go-ipfs-config" ) @@ -15,23 +17,25 @@ func TestToSubdomainURL(t *testing.T) { path string // out: url string - ok bool + err error }{ // DNSLink - {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", true}, + {"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil}, // Hostname with port - {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true}, + {"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil}, // CIDv0 → CIDv1base32 - {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true}, + {"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil}, + // CIDv1 with long sha512 + {"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, // PeerID as CIDv1 needs to have libp2p-key multicodec - {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true}, - {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true}, - // PeerID: ed25519+identity multihash - {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true}, + {"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", nil}, + {"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", nil}, + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil}, } { - url, ok := toSubdomainURL(test.hostname, test.path, r) - if ok != test.ok || url != test.url { - t.Errorf("(%s, %s) returned (%s, %t), expected (%s, %t)", test.hostname, test.path, url, ok, test.url, ok) + url, err := toSubdomainURL(test.hostname, test.path, r) + if url != test.url || !equalError(err, test.err) { + t.Errorf("(%s, %s) returned (%s, %v), expected (%s, %v)", test.hostname, test.path, url, err, test.url, test.err) } } } @@ -75,6 +79,30 @@ func TestPortStripping(t *testing.T) { } +func TestDNSPrefix(t *testing.T) { + for _, test := range []struct { + in string + out string + err error + }{ + // <= 63 + {"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil}, + {"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil}, + // > 63 + // PeerID: ed25519+identity multihash → CIDv1Base36 + {"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil}, + // CIDv1 with long sha512 → error + {"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")}, + } { + inCID, _ := cid.Decode(test.in) + out, err := toDNSPrefix(test.in, inCID) + if out != test.out || !equalError(err, test.err) { + t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err) + } + } + +} + func TestKnownSubdomainDetails(t *testing.T) { gwSpec := config.GatewaySpec{ UseSubdomains: true, @@ -150,3 +178,7 @@ func TestKnownSubdomainDetails(t *testing.T) { } } + +func equalError(a, b error) bool { + return a == nil && b == nil || a != nil && b != nil && a.Error() == b.Error() +} diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index 8b5133259af..acaa0637c26 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -119,7 +119,6 @@ test_expect_success "Publish test text file to IPNS" ' test_cmp expected2 output ' - # ensure we start with empty Gateway.PublicGateways test_expect_success 'start daemon with empty config for Gateway.PublicGateways' ' test_kill_ipfs_daemon && @@ -262,6 +261,7 @@ test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/ test_should_contain "subdir2-bar" list_response ' + # *.ipns.localhost # .ipns.localhost @@ -501,6 +501,43 @@ test_hostname_gateway_response_should_contain \ "http://127.0.0.1:$GWAY_PORT" \ "404 Not Found" +## ============================================================================ +## Special handling of CIDs that do not fit in a single DNS Label (>63chars) +## https://github.com/ipfs/go-ipfs/issues/7318 +## ============================================================================ + +# TODO: replace with cidv1 +# ed25519 fits under 63 char limit when represented in base36 +CIDv1_ED25519_RAW="12D3KooWP3ggTJV8LGckDHc4bVyXGhEWuBskoFyE6Rn2BJBqJtpa" +CIDv1_ED25519_DNSSAFE="k51qzi5uqu5dmcbvz6u9xahfqtsrky1g6tugpr5lqjyz0sc3j80muh0oi5tgtt" +# sha512 will be over 63char limit, even when represented in Base36 +CIDv1_TOO_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q) + +# local: *.localhost +test_localhost_gateway_response_should_contain \ + "request for a ED25519 CID at localhost/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ + "http://localhost:$GWAY_PORT/ipns/$CIDv1_ED25519_RAW" \ + "Location: http://${CIDv1_ED25519_DNSSAFE}.ipfs.localhost:$GWAY_PORT/" + +test_localhost_gateway_response_should_contain \ + "request for a too long CID at localhost/ipfs/{CIDv1} returns error" \ + "http://localhost:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ + "TODO: expect error" + +# public gateway: *.example.com + +test_hostname_gateway_response_should_contain \ + "request for a ED25519 CID at example.com/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_ED25519_RAW" \ + "Location: http://${CID_ED25519_DNSSAFE}.ipfs.example.com" + +test_hostname_gateway_response_should_contain \ + "request for a too long CID at example.com/ipfs/{CIDv1} returns error" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \ + "TODO: expect error" + ## ============================================================================ ## Test path-based requests with a custom hostname config