diff --git a/Makefile b/Makefile index fdec29df2..bf3342389 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,10 @@ provision-kubo: find ./fixtures -name '*.car' -exec ipfs dag import {} \; # tools +dnslink: + go build -o ./dnslinkgen ./dnslinkgen.go + ./dnslinkgen example.com > .env + fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index 93cc3f1b3..f5988d3f9 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -173,19 +173,19 @@ func main() { return err } - files, err := fixtures.List() + fxs, err := fixtures.List() if err != nil { return err } merged := cCtx.Bool("merged") if merged { - err = car.Merge(files, filepath.Join(directory, "fixtures.car")) + err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car")) if err != nil { return err } } else { - err = copyFiles(files, directory) + err = copyFiles(fxs.CarFiles, directory) if err != nil { return err } diff --git a/fixtures/t0109-dnslink.yml b/fixtures/t0109-dnslink.yml new file mode 100644 index 000000000..e83f56581 --- /dev/null +++ b/fixtures/t0109-dnslink.yml @@ -0,0 +1,17 @@ +# TODO: The dnslink definition below is equal to: +# REDIRECTS_DIR_CID=$(ipfs resolve -r /ipfs/$CAR_ROOT_CID/examples | cut -d "/" -f3) +# Hardcoding here seems like a good start, but the actual behavior is calling: +# fixture := car.MustOpenUnixfsCar("t0109-redirects.car") +# redirectDir := fixture.MustGetNode("examples") +# redirectDirCID := redirectDir.Base32Cid() +# fmt.Printf("/ipfs/%s", redirectDirCID) +dnslinks: + # list of id: key, value. + dnslink-enabled-on-fqdn: # this is the id, it can be reused through the test + # this is the name -> value. + # later it might be possible to rewrite the value on the left. + subdomain: dnslink-enabled-on-fqdn. + path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj + +# One tool will aggregate these dns link into `IPFS_NS_MAP="key:value" mapping. +# One tool will load this dns link into a fixture. diff --git a/go.mod b/go.mod index 07b4cc2cd..dcad3a64f 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/goleak v1.1.12 // indirect golang.org/x/sync v0.1.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -64,5 +65,6 @@ require ( golang.org/x/sys v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v2 v2.4.0 lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index ae56defbd..c5e07b4ee 100644 --- a/go.sum +++ b/go.sum @@ -318,9 +318,11 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index acb8ed651..db7124904 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -7,13 +7,13 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/dnslink" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestRedirectsFileSupport(t *testing.T) { fixture := car.MustOpenUnixfsCar("t0109-redirects.car") - redirectDir := fixture.MustGetNode("examples") redirectDirCID := redirectDir.Base32Cid() @@ -52,6 +52,7 @@ func TestRedirectsFileSupport(t *testing.T) { Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", Request: Request(). DoNotFollowRedirects(). + Header("Host", u.Host). URL("%s/redirect-one", redirectDirBaseURL), Response: Expect(). Status(301). @@ -311,7 +312,104 @@ func TestRedirectsFileSupport(t *testing.T) { } } -// TODO: dnslink tests +func TestRedirectsFileSupportWithDNSLink(t *testing.T) { + // TODO: load dnslink from the fixture + + // DNSLINK_FQDN="dnslink-enabled-on-fqdn.example.org" + // NO_DNSLINK_FQDN="dnslink-disabled-on-fqdn.example.com" + // noDnsLink := "dnslink-disabled-on-fqdn.example.com" + // export IPFS_NS_MAP="$DNSLINK_FQDN:/ipfs/$REDIRECTS_DIR_CID" + dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml") + dnsLink := dnsLinks.Get("dnslink-enabled-on-fqdn") + + gatewayURL := SubdomainGatewayURL + u, err := url.Parse(gatewayURL) + if err != nil { + t.Fatal(err) + } + + dnsLinkBaseUrl := fmt.Sprintf("%s://%s%s", u.Scheme, dnsLink, u.Host) + // noDnsLinkBaseUrl := fmt.Sprintf("%s://%s", u.Scheme, noDnsLink) + + tests := SugarTests{ + // # make sure test setup is valid (fail if CoreAPI is unable to resolve) + // test_expect_success "spoofed DNSLink record resolves in cli" " + // ipfs resolve /ipns/$DNSLINK_FQDN > result && + // test_should_contain \"$REDIRECTS_DIR_CID\" result && + // ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result && + // test_should_contain \"index.html\" result + // " + // SKIPPED + + // test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" ' + // curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // ' + { + Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + URL("%s/redirect-one", dnsLinkBaseUrl), + Response: Expect(). + Status(301). + Headers( + Header("Location", "/one.html"), + ), + }, + // # ensure custom 404 works and has the same cache headers as regular /ipns/ paths + // test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" ' + // curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response && + // test_should_contain "404 Not Found" response && + // test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response && + // test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_not_contain "immutable" response && + // test_should_contain "Date: " response && + // test_should_contain "my 404" response + // ' + { + Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file", + Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`, + Request: Request(). + URL("%s/not-found/has-no-redirects-entry", dnsLinkBaseUrl), + Response: Expect(). + Status(404). + Headers( + Header("Etag", "\"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\""), + Header("Cache-Control").Not().Contains("public, max-age=29030400, immutable"), + Header("Cache-Control").Not().Contains("immutable"), + Header("Date").Exists(), + ). + Body( + // TODO: I like the readble part here, maybe rewrite to load the file. + Contains("my 404"), + ), + }, + // test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" ' + // curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && + // test_should_not_contain "one.html" response && + // test_should_not_contain "301 Moved Permanently" response && + // test_should_not_contain "Location:" response + // ' + // TODO(lidel): this test seems to validate some kubo behavior not really gateway. + // { + // Name: "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled", + // Request: Request(). + // URL("%s/redirect-one", noDnsLinkBaseUrl), + // Response: Expect(). + // // TODO: add "status not equal to 301" check. + // // TODO: what `test_should_not_contain "one.html" response` actually means? No location correct? + // Headers( + // Header("Location").Not().Exists(), + // ), + // }, + } + + if specs.DNSLinkResolver.IsEnabled() { + Run(t, unwrapTests(t, tests).Build()) + } else { + t.Skip("subdomain gateway disabled") + } +} func unwrapTests(t *testing.T, tests SugarTests) SugarTests { t.Helper() diff --git a/tooling/dnslink/dnslink.go b/tooling/dnslink/dnslink.go new file mode 100644 index 000000000..55b9b6e77 --- /dev/null +++ b/tooling/dnslink/dnslink.go @@ -0,0 +1,52 @@ +package dnslink + +import ( + "fmt" + "os" + "path" + + "github.com/ipfs/gateway-conformance/tooling/fixtures" + "gopkg.in/yaml.v3" +) + +type DNSLinks struct { + DNSLinks map[string]DNSLink `yaml:"dnslinks"` +} + +type DNSLink struct { + Subdomain string `yaml:"subdomain"` + Path string `yaml:"path"` +} + +func OpenDNSLink(absPath string) (*DNSLinks, error) { + data, err := os.ReadFile(absPath) + if err != nil { + return nil, err + } + + var dnsLinks DNSLinks + err = yaml.Unmarshal(data, &dnsLinks) + if err != nil { + return nil, err + } + + return &dnsLinks, nil +} + +func MustOpenDNSLink(file string) *DNSLinks { + fixturePath := path.Join(fixtures.Dir(), file) + dnsLinks, err := OpenDNSLink(fixturePath) + if err != nil { + panic(err) + } + + return dnsLinks +} + +func (d *DNSLinks) Get(id string) string { + dnsLink, ok := d.DNSLinks[id] + if !ok { + panic(fmt.Errorf("dnslink %s not found", id)) + } + return dnsLink.Subdomain +} diff --git a/tooling/fixtures/list.go b/tooling/fixtures/list.go index d58cbec83..4151ff2d8 100644 --- a/tooling/fixtures/list.go +++ b/tooling/fixtures/list.go @@ -13,8 +13,14 @@ func Dir() string { return path.Join(home, "fixtures") } -func List() ([]string, error) { +type Fixtures struct { + CarFiles []string + ConfigFiles []string +} + +func List() (*Fixtures, error) { var carFiles []string + var yamlFiles []string err := filepath.WalkDir(Dir(), func(path string, d os.DirEntry, err error) error { if err != nil { @@ -31,6 +37,14 @@ func List() ([]string, error) { carFiles = append(carFiles, path) } + // if we have a yaml file, append: + if filepath.Ext(path) == ".yml" { + path, err := filepath.Abs(path) + if err != nil { + return err + } + yamlFiles = append(yamlFiles, path) + } return nil }) @@ -39,5 +53,8 @@ func List() ([]string, error) { return nil, err } - return carFiles, nil + return &Fixtures{ + CarFiles: carFiles, + ConfigFiles: yamlFiles, + }, nil } diff --git a/tooling/specs/specs.go b/tooling/specs/specs.go index a44ff42b5..4f00e5325 100644 --- a/tooling/specs/specs.go +++ b/tooling/specs/specs.go @@ -26,11 +26,13 @@ type Spec string const ( SubdomainGateway Spec = "subdomain-gateway" + DNSLinkResolver Spec = "dnslink-resolver" ) // All specs should be listed here. var specMaturity = map[Spec]maturity{ SubdomainGateway: stable, + DNSLinkResolver: stable, } func (s Spec) IsMature() bool {