Skip to content

Commit

Permalink
wip: support long CIDs in subdomains with TLS
Browse files Browse the repository at this point in the history 
This is WIP

Goal: add subdomain gateway support for CIDs longer than 63 characters
in a way that fits in a single DNS label to enable TLS for every root.

License: MIT
Signed-off-by: Marcin Rataj <lidel@lidel.org>
  • Loading branch information
@lidel
lidel committed Jun 29, 2020
1 parent 7ce1d75 commit 842ff8e
Show file tree
Hide file tree
Showing 3 changed files with 164 additions and 34 deletions.
105 changes: 83 additions & 22 deletions core/corehttp/hostname.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,12 +41,15 @@ var defaultKnownGateways = map[string]config.GatewaySpec{
"dweb.link": subdomainGatewaySpec,
}

// Label's max length in DNS (https://tools.ietf.org/html/rfc1034#page-7)
const dnsLabelMaxLength int = 63

// HostnameOption rewrites an incoming request based on the Host header.
func HostnameOption() ServeOption {
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
childMux := http.NewServeMux()

coreApi, err := coreapi.NewCoreAPI(n)
coreAPI, err := coreapi.NewCoreAPI(n)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -94,7 +97,12 @@ func HostnameOption() ServeOption {
if gw.UseSubdomains {
// Yes, redirect if applicable
// Example: dweb.link/ipfs/{cid} → {cid}.ipfs.dweb.link
if newURL, ok := toSubdomainURL(r.Host, r.URL.Path, r); ok {
newURL, err := toSubdomainURL(r.Host, r.URL.Path, r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if newURL != "" {
// Just to be sure single Origin can't be abused in
// web browsers that ignored the redirect for some
// reason, Clear-Site-Data header clears browsing
Expand Down Expand Up @@ -124,7 +132,7 @@ func HostnameOption() ServeOption {
// Not a whitelisted path

// Try DNSLink, if it was not explicitly disabled for the hostname
if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) {
if !gw.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) {
// rewrite path and handle as DNSLink
r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path
childMux.ServeHTTP(w, r)
Expand All @@ -151,16 +159,44 @@ func HostnameOption() ServeOption {
return
}

// Do we need to fix multicodec in PeerID represented as CIDv1?
if isPeerIDNamespace(ns) {
keyCid, err := cid.Decode(rootID)
if err == nil && keyCid.Type() != cid.Libp2pKey {
if newURL, ok := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r); ok {
// Redirect to CID fixed inside of toSubdomainURL()
// Check if rootID is a valid CID
if rootCID, err := cid.Decode(rootID); err == nil {
// Do we need to redirect root CID to a canonical DNS representation?
dnsCID, err := toDNSPrefix(rootID, rootCID)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if !strings.HasPrefix(r.Host, dnsCID) {
dnsPrefix := "/" + ns + "/" + dnsCID
newURL, err := toSubdomainURL(hostname, dnsPrefix+r.URL.Path, r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if newURL != "" {
// Redirect to deterministic CID to ensure CID
// always gets the same Origin on the web
http.Redirect(w, r, newURL, http.StatusMovedPermanently)
return
}
}

// Do we need to fix multicodec in PeerID represented as CIDv1?
if isPeerIDNamespace(ns) {
if rootCID.Type() != cid.Libp2pKey {
newURL, err := toSubdomainURL(hostname, pathPrefix+r.URL.Path, r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if newURL != "" {
// Redirect to CID fixed inside of toSubdomainURL()
http.Redirect(w, r, newURL, http.StatusMovedPermanently)
return
}
}
}
}

// Rewrite the path to not use subdomains
Expand All @@ -176,7 +212,7 @@ func HostnameOption() ServeOption {
// 1. is wildcard DNSLink enabled (Gateway.NoDNSLink=false)?
// 2. does Host header include a fully qualified domain name (FQDN)?
// 3. does DNSLink record exist in DNS?
if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreApi, r) {
if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(n.Context(), coreAPI, r) {
// rewrite path and handle as DNSLink
r.URL.Path = "/ipns/" + stripPort(r.Host) + r.URL.Path
childMux.ServeHTTP(w, r)
Expand Down Expand Up @@ -266,18 +302,38 @@ func isPeerIDNamespace(ns string) bool {
}
}

// Converts an identifier to DNS-safe representation that fits in 63 characters
func toDNSPrefix(rootID string, rootCID cid.Cid) (prefix string, err error) {
// Return as-is if things fit
if len(rootID) <= dnsLabelMaxLength {
return rootID, nil
}

// Convert to Base36 and see if that helped
rootID, err = cid.NewCidV1(rootCID.Type(), rootCID.Hash()).StringOfBase(mbase.Base36)
if err != nil {
return "", err
}
if len(rootID) <= dnsLabelMaxLength {
return rootID, nil
}

// Can't win with DNS at this point, return error
return "", fmt.Errorf("CID incompatible with DNS label length limit of 63: %s", rootID)
}

// Converts a hostname/path to a subdomain-based URL, if applicable.
func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok bool) {
func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, err error) {
var scheme, ns, rootID, rest string

query := r.URL.RawQuery
parts := strings.SplitN(path, "/", 4)
safeRedirectURL := func(in string) (out string, ok bool) {
safeRedirectURL := func(in string) (out string, err error) {
safeURI, err := url.ParseRequestURI(in)
if err != nil {
return "", false
return "", err
}
return safeURI.String(), true
return safeURI.String(), nil
}

// Support X-Forwarded-Proto if added by a reverse proxy
Expand All @@ -297,11 +353,11 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok
ns = parts[1]
rootID = parts[2]
default:
return "", false
return "", nil
}

if !isSubdomainNamespace(ns) {
return "", false
return "", nil
}

// add prefix if query is present
Expand All @@ -320,8 +376,8 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok
}

// If rootID is a CID, ensure it uses DNS-friendly text representation
if rootCid, err := cid.Decode(rootID); err == nil {
multicodec := rootCid.Type()
if rootCID, err := cid.Decode(rootID); err == nil {
multicodec := rootCID.Type()

// PeerIDs represented as CIDv1 are expected to have libp2p-key
// multicodec (https://github.com/libp2p/specs/pull/209).
Expand All @@ -334,11 +390,16 @@ func toSubdomainURL(hostname, path string, r *http.Request) (redirURL string, ok
// if object turns out to be a valid CID,
// ensure text representation used in subdomain is CIDv1 in Base32
// https://github.com/ipfs/in-web-browsers/issues/89
rootID, err = cid.NewCidV1(multicodec, rootCid.Hash()).StringOfBase(mbase.Base32)
rootCID = cid.NewCidV1(multicodec, rootCID.Hash())
rootID, err = rootCID.StringOfBase(mbase.Base32)
if err != nil {
return "", err
}

// make sure CID fits in DNS label
rootID, err = toDNSPrefix(rootID, rootCID)
if err != nil {
// should not error, but if it does, its clealy not possible to
// produce a subdomain URL
return "", false
return "", err
}
}

Expand Down
54 changes: 43 additions & 11 deletions core/corehttp/hostname_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package corehttp

import (
"errors"
"net/http/httptest"
"testing"

cid "github.com/ipfs/go-cid"
config "github.com/ipfs/go-ipfs-config"
)

Expand All @@ -15,23 +17,25 @@ func TestToSubdomainURL(t *testing.T) {
path string
// out:
url string
ok bool
err error
}{
// DNSLink
{"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", true},
{"localhost", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost/", nil},
// Hostname with port
{"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", true},
{"localhost:8080", "/ipns/dnslink.io", "http://dnslink.io.ipns.localhost:8080/", nil},
// CIDv0 → CIDv1base32
{"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", true},
{"localhost", "/ipfs/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "http://bafybeif7a7gdklt6hodwdrmwmxnhksctcuav6lfxlcyfz4khzl3qfmvcgu.ipfs.localhost/", nil},
// CIDv1 with long sha512
{"localhost", "/ipfs/bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")},
// PeerID as CIDv1 needs to have libp2p-key multicodec
{"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", true},
{"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", true},
// PeerID: ed25519+identity multihash
{"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://bafzaajaiaejcat4yhiwnr2qz73mtu6vrnj2krxlpfoa3wo2pllfi37quorgwh2jw.ipns.localhost/", true},
{"localhost", "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", "http://bafzbeieqhtl2l3mrszjnhv6hf2iloiitsx7mexiolcnywnbcrzkqxwslja.ipns.localhost/", nil},
{"localhost", "/ipns/bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "http://bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm.ipns.localhost/", nil},
// PeerID: ed25519+identity multihash → CIDv1Base36
{"localhost", "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", "http://k51qzi5uqu5di608geewp3nqkg0bpujoasmka7ftkyxgcm3fh1aroup0gsdrna.ipns.localhost/", nil},
} {
url, ok := toSubdomainURL(test.hostname, test.path, r)
if ok != test.ok || url != test.url {
t.Errorf("(%s, %s) returned (%s, %t), expected (%s, %t)", test.hostname, test.path, url, ok, test.url, ok)
url, err := toSubdomainURL(test.hostname, test.path, r)
if url != test.url || !equalError(err, test.err) {
t.Errorf("(%s, %s) returned (%s, %v), expected (%s, %v)", test.hostname, test.path, url, err, test.url, test.err)
}
}
}
Expand Down Expand Up @@ -75,6 +79,30 @@ func TestPortStripping(t *testing.T) {

}

func TestDNSPrefix(t *testing.T) {
for _, test := range []struct {
in string
out string
err error
}{
// <= 63
{"QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", nil},
{"bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", "bafybeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", nil},
// > 63
// PeerID: ed25519+identity multihash → CIDv1Base36
{"bafzaajaiaejca4syrpdu6gdx4wsdnokxkprgzxf4wrstuc34gxw5k5jrag2so5gk", "k51qzi5uqu5dj16qyiq0tajolkojyl9qdkr254920wxv7ghtuwcz593tp69z9m", nil},
// CIDv1 with long sha512 → error
{"bafkrgqe3ohjcjplc6n4f3fwunlj6upltggn7xqujbsvnvyw764srszz4u4rshq6ztos4chl4plgg4ffyyxnayrtdi5oc4xb2332g645433aeg", "", errors.New("CID incompatible with DNS label length limit of 63: kf1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5oj")},
} {
inCID, _ := cid.Decode(test.in)
out, err := toDNSPrefix(test.in, inCID)
if out != test.out || !equalError(err, test.err) {
t.Errorf("(%s): returned (%s, %v) expected (%s, %v)", test.in, out, err, test.out, test.err)
}
}

}

func TestKnownSubdomainDetails(t *testing.T) {
gwSpec := config.GatewaySpec{
UseSubdomains: true,
Expand Down Expand Up @@ -150,3 +178,7 @@ func TestKnownSubdomainDetails(t *testing.T) {
}

}

func equalError(a, b error) bool {
return a == nil && b == nil || a != nil && b != nil && a.Error() == b.Error()
}
39 changes: 38 additions & 1 deletion test/sharness/t0114-gateway-subdomains.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,6 @@ test_expect_success "Publish test text file to IPNS" '
test_cmp expected2 output
'


# ensure we start with empty Gateway.PublicGateways
test_expect_success 'start daemon with empty config for Gateway.PublicGateways' '
test_kill_ipfs_daemon &&
Expand Down Expand Up @@ -262,6 +261,7 @@ test_expect_success "request for deep path resource at {cid}.ipfs.localhost/sub/
test_should_contain "subdir2-bar" list_response
'


# *.ipns.localhost

# <libp2p-key>.ipns.localhost
Expand Down Expand Up @@ -501,6 +501,43 @@ test_hostname_gateway_response_should_contain \
"http://127.0.0.1:$GWAY_PORT" \
"404 Not Found"

## ============================================================================
## Special handling of CIDs that do not fit in a single DNS Label (>63chars)
## https://github.com/ipfs/go-ipfs/issues/7318
## ============================================================================

# TODO: replace with cidv1
# ed25519 fits under 63 char limit when represented in base36
CIDv1_ED25519_RAW="12D3KooWP3ggTJV8LGckDHc4bVyXGhEWuBskoFyE6Rn2BJBqJtpa"
CIDv1_ED25519_DNSSAFE="k51qzi5uqu5dmcbvz6u9xahfqtsrky1g6tugpr5lqjyz0sc3j80muh0oi5tgtt"
# sha512 will be over 63char limit, even when represented in Base36
CIDv1_TOO_LONG=$(echo $CID_VAL | ipfs add --cid-version 1 --hash sha2-512 -Q)

# local: *.localhost
test_localhost_gateway_response_should_contain \
"request for a ED25519 CID at localhost/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \
"http://localhost:$GWAY_PORT/ipns/$CIDv1_ED25519_RAW" \
"Location: http://${CIDv1_ED25519_DNSSAFE}.ipfs.localhost:$GWAY_PORT/"

test_localhost_gateway_response_should_contain \
"request for a too long CID at localhost/ipfs/{CIDv1} returns error" \
"http://localhost:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \
"TODO: expect error"

# public gateway: *.example.com

test_hostname_gateway_response_should_contain \
"request for a ED25519 CID at example.com/ipfs/{CIDv1} returns Location HTTP header for DNS-safe subdomain redirect in browsers" \
"example.com" \
"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_ED25519_RAW" \
"Location: http://${CID_ED25519_DNSSAFE}.ipfs.example.com"

test_hostname_gateway_response_should_contain \
"request for a too long CID at example.com/ipfs/{CIDv1} returns error" \
"example.com" \
"http://127.0.0.1:$GWAY_PORT/ipfs/$CIDv1_TOO_LONG" \
"TODO: expect error"


## ============================================================================
## Test path-based requests with a custom hostname config
Expand Down

0 comments on commit 842ff8e

Please sign in to comment.