Skip to content

Commit

Permalink
feat: draft dnslink support
Browse files Browse the repository at this point in the history
  • Loading branch information
laurentsenta committed Apr 5, 2023
1 parent 67d2a93 commit 266cdb6
Show file tree
Hide file tree
Showing 11 changed files with 294 additions and 7 deletions.
4 changes: 4 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ provision-kubo:
find ./fixtures -name '*.car' -exec ipfs dag import {} \;

# tools
dnslink:
go build -o ./dnslinkgen ./dnslinkgen.go
./dnslinkgen example.com > .env

fixtures.car: gateway-conformance
./gateway-conformance extract-fixtures --merged=true --dir=.

Expand Down
17 changes: 14 additions & 3 deletions cmd/gateway-conformance/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/car"
"github.com/ipfs/gateway-conformance/tooling/dnslink"
"github.com/ipfs/gateway-conformance/tooling/fixtures"
"github.com/urfave/cli/v2"
)
Expand Down Expand Up @@ -173,19 +174,29 @@ func main() {
return err
}

files, err := fixtures.List()
fxs, err := fixtures.List()
if err != nil {
return err
}

merged := cCtx.Bool("merged")
if merged {
err = car.Merge(files, filepath.Join(directory, "fixtures.car"))
err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car"))
if err != nil {
return err
}

err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json"))
if err != nil {
return err
}
} else {
err = copyFiles(files, directory)
err = copyFiles(fxs.CarFiles, directory)
if err != nil {
return err
}

err = copyFiles(fxs.ConfigFiles, directory)
if err != nil {
return err
}
Expand Down
39 changes: 39 additions & 0 deletions dnslinkgen.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package main

import (
"fmt"
"log"
"os"
"strings"

"github.com/ipfs/gateway-conformance/tooling/dnslink"
"github.com/ipfs/gateway-conformance/tooling/fixtures"
)

func main() {
if len(os.Args) < 2 {
fmt.Println("Usage: dnslinkgen <domain>")
os.Exit(1)
}

domain := os.Args[1]

fxs, err := fixtures.List()
if err != nil {
log.Fatal(err)
}

configs := fxs.ConfigFiles
aggMap, err := dnslink.Aggregate(configs)
if err != nil {
log.Fatal(err)
}

// print k=v on stdout
var kvs []string
for k, v := range aggMap {
kvs = append(kvs, fmt.Sprintf("%s%s:%s", k, domain, v))
}

fmt.Println("export IPFS_NS_MAP=\"" + strings.Join(kvs, ",") + "\"")
}
17 changes: 17 additions & 0 deletions fixtures/t0109-dnslink.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# TODO: The dnslink definition below is equal to:
# REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3)
# Hardcoding here seems like a good start, but the actual behavior is calling:
# fixture := car.MustOpenUnixfsCar("t0109-redirects.car")
# redirectDir := fixture.MustGetNode("examples")
# redirectDirCID := redirectDir.Base32Cid()
# fmt.Printf("/ipfs/%s", redirectDirCID)
dnslinks:
# list of id: key, value.
dnslink-enabled-on-fqdn: # this is the id, it can be reused through the test
# this is the name -> value.
# later it might be possible to rewrite the value on the left.
subdomain: dnslink-enabled-on-fqdn.
path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj

# One tool will aggregate these dns link into `IPFS_NS_MAP="key:value" mapping.
# One tool will load this dns link into a fixture.
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ require (
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.uber.org/goleak v1.1.12 // indirect
golang.org/x/sync v0.1.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

require (
Expand Down Expand Up @@ -64,5 +65,6 @@ require (
golang.org/x/sys v0.6.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/yaml.v2 v2.4.0
lukechampine.com/blake3 v1.1.7 // indirect
)
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -318,9 +318,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=
102 changes: 100 additions & 2 deletions tests/t0109_gateway_web_redirects_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ import (

"github.com/ipfs/gateway-conformance/tooling/car"
. "github.com/ipfs/gateway-conformance/tooling/check"
"github.com/ipfs/gateway-conformance/tooling/dnslink"
"github.com/ipfs/gateway-conformance/tooling/specs"
. "github.com/ipfs/gateway-conformance/tooling/test"
)

func TestRedirectsFileSupport(t *testing.T) {
fixture := car.MustOpenUnixfsCar("t0109-redirects.car")

redirectDir := fixture.MustGetNode("examples")
redirectDirCID := redirectDir.Base32Cid()

Expand Down Expand Up @@ -52,6 +52,7 @@ func TestRedirectsFileSupport(t *testing.T) {
Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file",
Request: Request().
DoNotFollowRedirects().
Header("Host", u.Host).
URL("%s/redirect-one", redirectDirBaseURL),
Response: Expect().
Status(301).
Expand Down Expand Up @@ -311,7 +312,104 @@ func TestRedirectsFileSupport(t *testing.T) {
}
}

// TODO: dnslink tests
func TestRedirectsFileSupportWithDNSLink(t *testing.T) {
// TODO: load dnslink from the fixture

// DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org"
// NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com"
// noDnsLink := "dnslink-disabled-on-fqdn.example.com"
// export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$REDIRECTS_DIR_CID"
dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml")
dnsLink := dnsLinks.Get("dnslink-enabled-on-fqdn")

gatewayURL := SubdomainGatewayURL
u, err := url.Parse(gatewayURL)
if err != nil {
t.Fatal(err)
}

dnsLinkBaseUrl := fmt.Sprintf("%s://%s%s", u.Scheme, dnsLink, u.Host)
// noDnsLinkBaseUrl := fmt.Sprintf("%s://%s", u.Scheme, noDnsLink)

tests := SugarTests{
// # make sure test setup is valid (fail if CoreAPI is unable to resolve)
// test_expect_success "spoofed DNSLink record resolves in cli" "
// ipfs resolve /ipns/$DNSLINK_FQDN > result &&
// test_should_contain \"$REDIRECTS_DIR_CID\" result &&
// ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result &&
// test_should_contain \"index.html\" result
// "
// SKIPPED

// test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" '
// curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
// test_should_contain "301 Moved Permanently" response &&
// test_should_contain "Location: /one.html" response
// '
{
Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file",
Request: Request().
URL("%s/redirect-one", dnsLinkBaseUrl),
Response: Expect().
Status(301).
Headers(
Header("Location", "/one.html"),
),
},
// # ensure custom 404 works and has the same cache headers as regular /ipns/ paths
// test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" '
// curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response &&
// test_should_contain "404 Not Found" response &&
// test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response &&
// test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response &&
// test_should_not_contain "immutable" response &&
// test_should_contain "Date: " response &&
// test_should_contain "my 404" response
// '
{
Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file",
Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`,
Request: Request().
URL("%s/not-found/has-no-redirects-entry", dnsLinkBaseUrl),
Response: Expect().
Status(404).
Headers(
Header("Etag", "\"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\""),
Header("Cache-Control").Not().Contains("public, max-age=29030400, immutable"),
Header("Cache-Control").Not().Contains("immutable"),
Header("Date").Exists(),
).
Body(
// TODO: I like the readble part here, maybe rewrite to load the file.
Contains("my 404"),
),
},
// test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" '
// curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
// test_should_not_contain "one.html" response &&
// test_should_not_contain "301 Moved Permanently" response &&
// test_should_not_contain "Location:" response
// '
// TODO(lidel): this test seems to validate some kubo behavior not really gateway.
// {
// Name: "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled",
// Request: Request().
// URL("%s/redirect-one", noDnsLinkBaseUrl),
// Response: Expect().
// // TODO: add "status not equal to 301" check.
// // TODO: what `test_should_not_contain "one.html" response` actually means? No location correct?
// Headers(
// Header("Location").Not().Exists(),
// ),
// },
}

if specs.DNSLinkResolver.IsEnabled() {
Run(t, unwrapTests(t, tests).Build())
} else {
t.Skip("subdomain gateway disabled")
}
}

func unwrapTests(t *testing.T, tests SugarTests) SugarTests {
t.Helper()
Expand Down
52 changes: 52 additions & 0 deletions tooling/dnslink/dnslink.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package dnslink

import (
"fmt"
"os"
"path"

"github.com/ipfs/gateway-conformance/tooling/fixtures"
"gopkg.in/yaml.v3"
)

type DNSLinks struct {
DNSLinks map[string]DNSLink `yaml:"dnslinks"`
}

type DNSLink struct {
Subdomain string `yaml:"subdomain"`
Path string `yaml:"path"`
}

func OpenDNSLink(absPath string) (*DNSLinks, error) {
data, err := os.ReadFile(absPath)
if err != nil {
return nil, err
}

var dnsLinks DNSLinks
err = yaml.Unmarshal(data, &dnsLinks)
if err != nil {
return nil, err
}

return &dnsLinks, nil
}

func MustOpenDNSLink(file string) *DNSLinks {
fixturePath := path.Join(fixtures.Dir(), file)
dnsLinks, err := OpenDNSLink(fixturePath)
if err != nil {
panic(err)
}

return dnsLinks
}

func (d *DNSLinks) Get(id string) string {
dnsLink, ok := d.DNSLinks[id]
if !ok {
panic(fmt.Errorf("dnslink %s not found", id))
}
return dnsLink.Subdomain
}
43 changes: 43 additions & 0 deletions tooling/dnslink/merge.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package dnslink

import (
"encoding/json"
"fmt"
"os"
)

func Aggregate(inputPaths []string) (map[string]string, error) {
aggMap := make(map[string]string)

for _, file := range inputPaths {
dnsLinks, err := OpenDNSLink(file)
if err != nil {
return nil, fmt.Errorf("error loading file %s: %v", file, err)
}

for _, link := range dnsLinks.DNSLinks {
if _, ok := aggMap[link.Subdomain]; ok {
return nil, fmt.Errorf("collision detected for subdomain %s", link.Subdomain)
}

aggMap[link.Subdomain] = link.Path
}
}

return aggMap, nil
}

func Merge(inputPaths []string, outputPath string) error {
kvs, err := Aggregate(inputPaths)
if err != nil {
return err
}

j, err := json.MarshalIndent(kvs, "", " ")
if err != nil {
return err
}

err = os.WriteFile(outputPath, j, 0644)
return err
}
Loading

0 comments on commit 266cdb6

Please sign in to comment.