feat: draft dnslink support

Makefile

@@ -18,6 +18,10 @@ provision-kubo:
 	find ./fixtures -name '*.car' -exec ipfs dag import {} \;
 
 # tools
+dnslink:
+	go build -o ./dnslinkgen ./dnslinkgen.go
+	./dnslinkgen example.com > .env
+
 fixtures.car: gateway-conformance
 	./gateway-conformance extract-fixtures --merged=true --dir=.
 

cmd/gateway-conformance/main.go

@@ -173,19 +173,19 @@ func main() {
 						return err
 					}
 
-					files, err := fixtures.List()
+					fxs, err := fixtures.List()
 					if err != nil {
 						return err
 					}
 
 					merged := cCtx.Bool("merged")
 					if merged {
-						err = car.Merge(files, filepath.Join(directory, "fixtures.car"))
+						err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car"))
 						if err != nil {
 							return err
 						}
 					} else {
-						err = copyFiles(files, directory)
+						err = copyFiles(fxs.CarFiles, directory)
 						if err != nil {
 							return err
 						}

fixtures/t0109-dnslink.yml

@@ -0,0 +1,17 @@
+# TODO: The dnslink definition below is equal to:
+# REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3)
+# Hardcoding here seems like a good start, but the actual behavior is calling:
+# fixture := car.MustOpenUnixfsCar("t0109-redirects.car")
+# redirectDir := fixture.MustGetNode("examples")
+# redirectDirCID := redirectDir.Base32Cid()
+# fmt.Printf("/ipfs/%s", redirectDirCID)
+dnslinks:
+  # list of id: key, value.
+  dnslink-enabled-on-fqdn: # this is the id, it can be reused through the test
+    # this is the name -> value.
+    # later it might be possible to rewrite the value on the left.
+    subdomain: dnslink-enabled-on-fqdn.
+    path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj
+
+# One tool will aggregate these dns link into `IPFS_NS_MAP="key:value" mapping.
+# One tool will load this dns link into a fixture.

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.uber.org/goleak v1.1.12 // indirect
 	golang.org/x/sync v0.1.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (
@@ -64,5 +65,6 @@ require (
 	golang.org/x/sys v0.6.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	lukechampine.com/blake3 v1.1.7 // indirect
 )

go.sum

@@ -318,9 +318,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0=
 lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA=

tests/t0109_gateway_web_redirects_test.go

@@ -7,13 +7,13 @@ import (
 
 	"github.com/ipfs/gateway-conformance/tooling/car"
 	. "github.com/ipfs/gateway-conformance/tooling/check"
+	"github.com/ipfs/gateway-conformance/tooling/dnslink"
 	"github.com/ipfs/gateway-conformance/tooling/specs"
 	. "github.com/ipfs/gateway-conformance/tooling/test"
 )
 
 func TestRedirectsFileSupport(t *testing.T) {
 	fixture := car.MustOpenUnixfsCar("t0109-redirects.car")
-
 	redirectDir := fixture.MustGetNode("examples")
 	redirectDirCID := redirectDir.Base32Cid()
 
@@ -52,6 +52,7 @@ func TestRedirectsFileSupport(t *testing.T) {
 				Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file",
 				Request: Request().
 					DoNotFollowRedirects().
+					Header("Host", u.Host).
 					URL("%s/redirect-one", redirectDirBaseURL),
 				Response: Expect().
 					Status(301).
@@ -311,7 +312,104 @@ func TestRedirectsFileSupport(t *testing.T) {
 	}
 }
 
-// TODO: dnslink tests
+func TestRedirectsFileSupportWithDNSLink(t *testing.T) {
+	// TODO: load dnslink from the fixture
+
+	// DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org"
+	// NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com"
+	// noDnsLink := "dnslink-disabled-on-fqdn.example.com"
+	// export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$REDIRECTS_DIR_CID"
+	dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml")
+	dnsLink := dnsLinks.Get("dnslink-enabled-on-fqdn")
+
+	gatewayURL := SubdomainGatewayURL
+	u, err := url.Parse(gatewayURL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	dnsLinkBaseUrl := fmt.Sprintf("%s://%s%s", u.Scheme, dnsLink, u.Host)
+	// noDnsLinkBaseUrl := fmt.Sprintf("%s://%s", u.Scheme, noDnsLink)
+
+	tests := SugarTests{
+		// # make sure test setup is valid (fail if CoreAPI is unable to resolve)
+		// test_expect_success "spoofed DNSLink record resolves in cli" "
+		// ipfs resolve /ipns/$DNSLINK_FQDN > result &&
+		// test_should_contain \"$REDIRECTS_DIR_CID\" result &&
+		// ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result &&
+		// test_should_contain \"index.html\" result
+		// "
+		// SKIPPED
+
+		// test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" '
+		// curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
+		// test_should_contain "301 Moved Permanently" response &&
+		// test_should_contain "Location: /one.html" response
+		// '
+		{
+			Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file",
+			Request: Request().
+				URL("%s/redirect-one", dnsLinkBaseUrl),
+			Response: Expect().
+				Status(301).
+				Headers(
+					Header("Location", "/one.html"),
+				),
+		},
+		// # ensure custom 404 works and has the same cache headers as regular /ipns/ paths
+		// test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" '
+		// curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response &&
+		// test_should_contain "404 Not Found" response &&
+		// test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response &&
+		// test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response &&
+		// test_should_not_contain "immutable" response &&
+		// test_should_contain "Date: " response &&
+		// test_should_contain "my 404" response
+		// '
+		{
+			Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file",
+			Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`,
+			Request: Request().
+				URL("%s/not-found/has-no-redirects-entry", dnsLinkBaseUrl),
+			Response: Expect().
+				Status(404).
+				Headers(
+					Header("Etag", "\"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\""),
+					Header("Cache-Control").Not().Contains("public, max-age=29030400, immutable"),
+					Header("Cache-Control").Not().Contains("immutable"),
+					Header("Date").Exists(),
+				).
+				Body(
+					// TODO: I like the readble part here, maybe rewrite to load the file.
+					Contains("my 404"),
+				),
+		},
+		// test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" '
+		// curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response &&
+		// test_should_not_contain "one.html" response &&
+		// test_should_not_contain "301 Moved Permanently" response &&
+		// test_should_not_contain "Location:" response
+		// '
+		// TODO(lidel): this test seems to validate some kubo behavior not really gateway.
+		// {
+		// 	Name: "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled",
+		// 	Request: Request().
+		// 		URL("%s/redirect-one", noDnsLinkBaseUrl),
+		// 	Response: Expect().
+		// 		// TODO: add "status not equal to 301" check.
+		// 		// TODO: what `test_should_not_contain "one.html" response` actually means? No location correct?
+		// 		Headers(
+		// 			Header("Location").Not().Exists(),
+		// 		),
+		// },
+	}
+
+	if specs.DNSLinkResolver.IsEnabled() {
+		Run(t, unwrapTests(t, tests).Build())
+	} else {
+		t.Skip("subdomain gateway disabled")
+	}
+}
 
 func unwrapTests(t *testing.T, tests SugarTests) SugarTests {
 	t.Helper()

tooling/dnslink/dnslink.go

@@ -0,0 +1,52 @@
+package dnslink
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/ipfs/gateway-conformance/tooling/fixtures"
+	"gopkg.in/yaml.v3"
+)
+
+type DNSLinks struct {
+	DNSLinks map[string]DNSLink `yaml:"dnslinks"`
+}
+
+type DNSLink struct {
+	Subdomain string `yaml:"subdomain"`
+	Path      string `yaml:"path"`
+}
+
+func OpenDNSLink(absPath string) (*DNSLinks, error) {
+	data, err := os.ReadFile(absPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var dnsLinks DNSLinks
+	err = yaml.Unmarshal(data, &dnsLinks)
+	if err != nil {
+		return nil, err
+	}
+
+	return &dnsLinks, nil
+}
+
+func MustOpenDNSLink(file string) *DNSLinks {
+	fixturePath := path.Join(fixtures.Dir(), file)
+	dnsLinks, err := OpenDNSLink(fixturePath)
+	if err != nil {
+		panic(err)
+	}
+
+	return dnsLinks
+}
+
+func (d *DNSLinks) Get(id string) string {
+	dnsLink, ok := d.DNSLinks[id]
+	if !ok {
+		panic(fmt.Errorf("dnslink %s not found", id))
+	}
+	return dnsLink.Subdomain
+}

tooling/fixtures/list.go

@@ -13,8 +13,14 @@ func Dir() string {
 	return path.Join(home, "fixtures")
 }
 
-func List() ([]string, error) {
+type Fixtures struct {
+	CarFiles    []string
+	ConfigFiles []string
+}
+
+func List() (*Fixtures, error) {
 	var carFiles []string
+	var yamlFiles []string
 
 	err := filepath.WalkDir(Dir(), func(path string, d os.DirEntry, err error) error {
 		if err != nil {
@@ -31,6 +37,14 @@ func List() ([]string, error) {
 
 			carFiles = append(carFiles, path)
 		}
+		// if we have a yaml file, append:
+		if filepath.Ext(path) == ".yml" {
+			path, err := filepath.Abs(path)
+			if err != nil {
+				return err
+			}
+			yamlFiles = append(yamlFiles, path)
+		}
 
 		return nil
 	})
@@ -39,5 +53,8 @@ func List() ([]string, error) {
 		return nil, err
 	}
 
-	return carFiles, nil
+	return &Fixtures{
+		CarFiles:    carFiles,
+		ConfigFiles: yamlFiles,
+	}, nil
 }

tooling/specs/specs.go

@@ -26,11 +26,13 @@ type Spec string
 
 const (
 	SubdomainGateway Spec = "subdomain-gateway"
+	DNSLinkResolver Spec = "dnslink-resolver"
 )
 
 // All specs should be listed here.
 var specMaturity = map[Spec]maturity{
 	SubdomainGateway: stable,
+	DNSLinkResolver: stable,
 }
 
 func (s Spec) IsMature() bool {