From 36368ee4ddfb8943d96f2e34b3283d11a1498ff6 Mon Sep 17 00:00:00 2001 From: Dennis Trautwein Date: Sat, 5 Dec 2020 17:02:40 +0100 Subject: [PATCH 1/5] feat: support requests from registerProtocolHandler This commit adds support for requests produced by navigator.registerProtocolHandler on gateways. Now one can register `dweb.link` as an URI handler for `ipfs://`: ``` navigator.registerProtocolHandler('ipfs', 'https://dweb.link/ipfs/?uri=%s', 'ipfs resolver') ``` Then opening `ipfs://QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR` will produce an HTTP GET call to: ``` https://dweb.link/ipfs?uri=ipfs%3A%2F%2FQmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR ``` The query parameter `uri` will now be parsed and the given content identifier resolved via: `https://dweb.link/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR` --- core/corehttp/gateway_handler.go | 17 +++++++++++++ core/corehttp/gateway_test.go | 41 ++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index b78bacb02e5..8c34ba4541e 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -181,6 +181,23 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request } originalUrlPath := prefix + requestURI.Path + // Query parameter handling to support requests produced by navigator.registerProtocolHandler. + // E.g. This code will redirect calls to /ipfs/?uri=ipfs%3A%2F%2Fcontent-identifier + // to /ipfs/content-identifier. + if uri := r.URL.Query().Get("uri"); uri != "" { + u, err := url.Parse(uri) + if err != nil { + webError(w, "failed to parse uri query parameter", err, http.StatusBadRequest) + return + } + if u.Scheme != "ipfs" && u.Scheme != "ipns" { + webError(w, "uri query parameter scheme must be ipfs or ipns", err, http.StatusBadRequest) + return + } + http.Redirect(w, r, gopath.Join("/", prefix, u.Scheme, u.Host), http.StatusMovedPermanently) + return + } + // Service Worker registration request if r.Header.Get("Service-Worker") == "script" { // Disallow Service Worker registration on namespace roots diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index f98b4a7737b..049ccd27c4c 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -161,6 +161,47 @@ func matchPathOrBreadcrumbs(s string, expected string) bool { return matched } +func TestUriQueryRedirect(t *testing.T) { + ts, _, _ := newTestServerAndNode(t, mockNamesys{}) + + cid := "QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR" + for i, test := range []struct { + path string + status int + location string + }{ + {"/ipfs/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipfs/?uri=ipfs%3A%2F%2F" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipfs?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/?uri=ipfs://" + cid}, + {"/ipfs/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, + {"/ipns?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/?uri=ipns://" + cid}, + {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipfs/?uri=unsupported://" + cid, http.StatusBadRequest, ""}, + {"/ipfs/?uri=" + cid, http.StatusBadRequest, ""}, + } { + + r, err := http.NewRequest(http.MethodGet, ts.URL+test.path, nil) + if err != nil { + t.Fatal(err) + } + resp, err := doWithoutRedirect(r) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + + if resp.StatusCode != test.status { + t.Errorf("(%d) got %d, expected %d from %s", i, resp.StatusCode, test.status, ts.URL+test.path) + } + + locHdr := resp.Header.Get("Location") + if locHdr != test.location { + t.Errorf("(%d) location header got %s, expected %s from %s", i, locHdr, test.location, ts.URL+test.path) + } + } +} + func TestGatewayGet(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) From 3de5b14e0c248ab1f611215836ace4e134a25b61 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 12 Dec 2020 02:43:16 +0100 Subject: [PATCH 2/5] fix: ?uri= url-decode and preserve query This makes ?uri= param able to process URIs passed by web browsers https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler --- core/corehttp/gateway_handler.go | 21 ++++++++++++++++----- core/corehttp/gateway_test.go | 10 ++++++++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 8c34ba4541e..2ef4fefb1b8 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -181,10 +181,17 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request } originalUrlPath := prefix + requestURI.Path - // Query parameter handling to support requests produced by navigator.registerProtocolHandler. - // E.g. This code will redirect calls to /ipfs/?uri=ipfs%3A%2F%2Fcontent-identifier - // to /ipfs/content-identifier. - if uri := r.URL.Query().Get("uri"); uri != "" { + // ?uri query param support for requests produced by web browsers + // via navigator.registerProtocolHandler Web API + // https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler + // TLDR: redirect /ipfs/?uri=ipfs%3A%2F%2Fcid%3Fquery%3Dval to /ipfs/cid?query=val + if uriParam := r.URL.Query().Get("uri"); uriParam != "" { + // Browsers will pass URI in URL-escaped form, we need to unescape it first + uri, err := url.QueryUnescape(uriParam) + if err != nil { + webError(w, "failed to unescape uri query parameter", err, http.StatusBadRequest) + return + } u, err := url.Parse(uri) if err != nil { webError(w, "failed to parse uri query parameter", err, http.StatusBadRequest) @@ -194,7 +201,11 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request webError(w, "uri query parameter scheme must be ipfs or ipns", err, http.StatusBadRequest) return } - http.Redirect(w, r, gopath.Join("/", prefix, u.Scheme, u.Host), http.StatusMovedPermanently) + path := u.Path + if u.RawQuery != "" { // preserve query if present + path = path + "?" + u.RawQuery + } + http.Redirect(w, r, gopath.Join("/", prefix, u.Scheme, u.Host, path), http.StatusMovedPermanently) return } diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index 049ccd27c4c..d4fcf5a3825 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -170,12 +170,18 @@ func TestUriQueryRedirect(t *testing.T) { status int location string }{ + // - Browsers will send original URI in URL-escaped form + // - We expect query parameters to be persisted + // - We drop fragments, as those should not be sent by a browser + {"/ipfs/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, + {"/ipfs/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, {"/ipfs/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, - {"/ipfs/?uri=ipfs%3A%2F%2F" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, {"/ipfs?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/?uri=ipfs://" + cid}, {"/ipfs/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, + {"/ipns/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html%23header-%C4%85", http.StatusMovedPermanently, "/ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, + {"/ipns/?uri=ipns%3A%2F%2Fexample.com%2Fwiki%2FFoo_%C4%85%C4%99.html%3Ffilename%3Dtest-%C4%99.html", http.StatusMovedPermanently, "/ipns/example.com/wiki/Foo_%c4%85%c4%99.html?filename=test-%c4%99.html"}, {"/ipns?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/?uri=ipns://" + cid}, - {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, + {"/ipns/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, {"/ipfs/?uri=unsupported://" + cid, http.StatusBadRequest, ""}, {"/ipfs/?uri=" + cid, http.StatusBadRequest, ""}, From abb25a1cfc202ab4427bb1d3fa0c16d6e2700bd5 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 12 Dec 2020 12:28:42 +0100 Subject: [PATCH 3/5] refactor: remove redundant urlescape URL.Query() will already decode the query parameters --- core/corehttp/gateway_handler.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 2ef4fefb1b8..e4719817418 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -186,13 +186,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler // TLDR: redirect /ipfs/?uri=ipfs%3A%2F%2Fcid%3Fquery%3Dval to /ipfs/cid?query=val if uriParam := r.URL.Query().Get("uri"); uriParam != "" { - // Browsers will pass URI in URL-escaped form, we need to unescape it first - uri, err := url.QueryUnescape(uriParam) - if err != nil { - webError(w, "failed to unescape uri query parameter", err, http.StatusBadRequest) - return - } - u, err := url.Parse(uri) + u, err := url.Parse(uriParam) if err != nil { webError(w, "failed to parse uri query parameter", err, http.StatusBadRequest) return From cdad39479cb0f08e8a9c15be0009427b63c7e73f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 13 Jan 2021 16:41:38 +0100 Subject: [PATCH 4/5] test: sharness for ?uri= router Additional tests to ensure there are no regressions, as this will be used by browser vendors in the future. --- test/sharness/t0114-gateway-subdomains.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0114-gateway-subdomains.sh b/test/sharness/t0114-gateway-subdomains.sh index 4c5f49b6351..b9af0805e57 100755 --- a/test/sharness/t0114-gateway-subdomains.sh +++ b/test/sharness/t0114-gateway-subdomains.sh @@ -379,7 +379,12 @@ test_expect_success "request for http://example.com/ipfs/{CID} with X-Forwarded- test_should_contain \"Location: https://$CIDv1.ipfs.example.com/\" response " - +# Support ipfs:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler +test_hostname_gateway_response_should_contain \ + "request for example.com/ipfs/?uri=ipfs%3A%2F%2F.. produces redirect to /ipfs/.. content path" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipfs/?uri=ipfs%3A%2F%2FQmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco%2Fwiki%2FDiego_Maradona.html" \ + "Location: /ipfs/QmXoypizjW3WknFiJnKLwHCnL72vedxjQkDDP1mXWo6uco/wiki/Diego_Maradona.html" # example.com/ipns/ @@ -411,6 +416,13 @@ test_expect_success \ test_should_contain \"Location: https://en-wikipedia--on--ipfs-org.ipns.example.com/wiki\" response " +# Support ipns:// in https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler +test_hostname_gateway_response_should_contain \ + "request for example.com/ipns/?uri=ipns%3A%2F%2F.. produces redirect to /ipns/.. content path" \ + "example.com" \ + "http://127.0.0.1:$GWAY_PORT/ipns/?uri=ipns%3A%2F%2Fen.wikipedia-on-ipfs.org" \ + "Location: /ipns/en.wikipedia-on-ipfs.org" + # *.ipfs.example.com: subdomain requests made with custom FQDN in Host header test_hostname_gateway_response_should_contain \ From a0f90d3a140a4616c5930bcb8c62445717ccc857 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Thu, 14 Jan 2021 20:51:02 +0100 Subject: [PATCH 5/5] test: cover 2 remaining lines --- core/corehttp/gateway_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index d4fcf5a3825..a05e456ab6c 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -184,6 +184,7 @@ func TestUriQueryRedirect(t *testing.T) { {"/ipns/?uri=ipns://" + cid, http.StatusMovedPermanently, "/ipns/" + cid}, {"/ipns/?uri=ipfs://" + cid, http.StatusMovedPermanently, "/ipfs/" + cid}, {"/ipfs/?uri=unsupported://" + cid, http.StatusBadRequest, ""}, + {"/ipfs/?uri=invaliduri", http.StatusBadRequest, ""}, {"/ipfs/?uri=" + cid, http.StatusBadRequest, ""}, } {