diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51777a329..6d626f03c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,7 +26,10 @@ jobs: - name: Configure Kubo Gateway run: | ipfs init; - ./gateway-conformance/kubo-config.example.sh; + source ./gateway-conformance/kubo-config.example.sh; + IPFS_NS_MAP=$(cat ./fixtures/dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') + echo "IPFS_NS_MAP=${IPFS_NS_MAP}" >> $GITHUB_ENV + # note: the IPFS_NS_MAP set above will be passed the daemon: - uses: ipfs/start-ipfs-daemon-action@v1 - name: Provision Kubo Gateway run: | diff --git a/Makefile b/Makefile index fdec29df2..02cb059f4 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,9 @@ provision-cargateway: ./fixtures.car # cd go-libipfs/examples/car && go install car -c ./fixtures.car & -provision-kubo: +provision-kubo: fixtures.car find ./fixtures -name '*.car' -exec ipfs dag import {} \; -# tools fixtures.car: gateway-conformance ./gateway-conformance extract-fixtures --merged=true --dir=. diff --git a/cmd/gateway-conformance/main.go b/cmd/gateway-conformance/main.go index 93cc3f1b3..a56ac45bc 100644 --- a/cmd/gateway-conformance/main.go +++ b/cmd/gateway-conformance/main.go @@ -12,6 +12,7 @@ import ( "github.com/ipfs/gateway-conformance/tooling" "github.com/ipfs/gateway-conformance/tooling/car" + "github.com/ipfs/gateway-conformance/tooling/dnslink" "github.com/ipfs/gateway-conformance/tooling/fixtures" "github.com/urfave/cli/v2" ) @@ -173,19 +174,34 @@ func main() { return err } - files, err := fixtures.List() + fxs, err := fixtures.List() if err != nil { return err } merged := cCtx.Bool("merged") if merged { - err = car.Merge(files, filepath.Join(directory, "fixtures.car")) + err = car.Merge(fxs.CarFiles, filepath.Join(directory, "fixtures.car")) + if err != nil { + return err + } + + err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) if err != nil { return err } } else { - err = copyFiles(files, directory) + err = copyFiles(fxs.CarFiles, directory) + if err != nil { + return err + } + + err = copyFiles(fxs.ConfigFiles, directory) + if err != nil { + return err + } + + err := dnslink.Merge(fxs.ConfigFiles, filepath.Join(directory, "dnslinks.json")) if err != nil { return err } diff --git a/fixtures/fixture.schema.json b/fixtures/fixture.schema.json new file mode 100644 index 000000000..ae26a75bf --- /dev/null +++ b/fixtures/fixture.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "dnslinks": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "subdomain": { + "type": "string" + }, + "path": { + "type": "string" + } + }, + "required": [ + "subdomain", + "path" + ], + "additionalProperties": false + } + } + }, + "additionalProperties": false +} \ No newline at end of file diff --git a/fixtures/t0109-dnslink.yml b/fixtures/t0109-dnslink.yml new file mode 100644 index 000000000..d65b5f638 --- /dev/null +++ b/fixtures/t0109-dnslink.yml @@ -0,0 +1,7 @@ +# yaml-language-server: $schema=fixture.schema.json +dnslinks: + custom-dnslink: + subdomain: dnslink-enabled-on-fqdn + # this is the cid of the folder + # t0109-redirects.car:/examples/ + path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj \ No newline at end of file diff --git a/go.mod b/go.mod index 8088f2391..9fd9b449d 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/ipld/go-ipld-prime v0.20.0 github.com/stretchr/testify v1.8.2 github.com/urfave/cli/v2 v2.25.3 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -21,7 +22,6 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect go.uber.org/goleak v1.1.12 // indirect golang.org/x/sync v0.1.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index dad4a8d01..f18a3caf0 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,4 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a h1:E/8AP5dFtMhl5KPJz66Kt9G0n+7Sn41Fy1wv9/jHOrc= -github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -21,8 +19,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -49,8 +45,6 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc= -github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/boxo v0.8.1 h1:3DkKBCK+3rdEB5t77WDShUXXhktYwH99mkAsgajsKrU= github.com/ipfs/boxo v0.8.1/go.mod h1:xJ2hVb4La5WyD7GvKYE0lq2g1rmQZoCD2K4WNrV6aZI= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= @@ -65,8 +59,6 @@ github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUP github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= -github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.6.0 h1:JKyz+Gvz1QEZw0LsX1IBn+JFCJQH4SJVFtM4uWU0Myk= @@ -115,8 +107,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= @@ -137,8 +127,6 @@ github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0 github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -164,12 +152,8 @@ github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2 github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= -github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -226,8 +210,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.25.0 h1:ykdZKuQey2zq0yin/l7JOm9Mh+pg72ngYMeB0ABn6q8= -github.com/urfave/cli/v2 v2.25.0/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/urfave/cli/v2 v2.25.3 h1:VJkt6wvEBOoSjPFQvOkv6iWIrsJyCrKGtCtxXWwmGeY= github.com/urfave/cli/v2 v2.25.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= @@ -238,8 +220,6 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0 h1:XYEgH2nJgsrcrj32p+SAbx6T3s/6QknOXezXtz7kzbg= github.com/whyrusleeping/cbor-gen v0.0.0-20230418232409-daab9ece03a0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= @@ -248,12 +228,8 @@ github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsr github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opentelemetry.io/otel v1.14.0 h1:/79Huy8wbf5DnIPhemGB+zEPVwnN6fuQybr/SRXa6hM= -go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188WlggnNeU= go.opentelemetry.io/otel v1.15.0 h1:NIl24d4eiLJPM0vKn4HjLYM+UZf6gSfi9Z+NmCxkWbk= go.opentelemetry.io/otel v1.15.0/go.mod h1:qfwLEbWhLPk5gyWrne4XnF0lC8wtywbuJbgfAE3zbek= -go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= -go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/otel/trace v1.15.0 h1:5Fwje4O2ooOxkfyqI/kJwxWotggDLix4BSAvpE1wlpo= go.opentelemetry.io/otel/trace v1.15.0/go.mod h1:CUsmE2Ht1CRkvE8OsMESvraoZrrcgD1J2W8GV1ev0Y4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -265,8 +241,6 @@ go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= @@ -281,12 +255,8 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w= -golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -301,7 +271,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -319,8 +289,6 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -346,8 +314,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/kubo-config.example.sh b/kubo-config.example.sh index d1873534a..14cf41969 100755 --- a/kubo-config.example.sh +++ b/kubo-config.example.sh @@ -9,4 +9,9 @@ ipfs config --json Gateway.PublicGateways '{ "InlineDNSLink": true, "Paths": ["/ipfs", "/ipns", "/api"] } -}' \ No newline at end of file +}' + +export IPFS_NS_MAP=$(cat ./dnslinks.json | jq -r 'to_entries | map("\(.key).example.com:\(.value)") | join(",")') + +echo "Set the following IPFS_NS_MAP before starting the kubo daemon:" +echo "IPFS_NS_MAP=${IPFS_NS_MAP}" diff --git a/tests/t0109_gateway_web_redirects_test.go b/tests/t0109_gateway_web_redirects_test.go index e5471acf0..ad7aed5da 100644 --- a/tests/t0109_gateway_web_redirects_test.go +++ b/tests/t0109_gateway_web_redirects_test.go @@ -7,13 +7,14 @@ import ( "github.com/ipfs/gateway-conformance/tooling/car" . "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/dnslink" + "github.com/ipfs/gateway-conformance/tooling/helpers" "github.com/ipfs/gateway-conformance/tooling/specs" . "github.com/ipfs/gateway-conformance/tooling/test" ) func TestRedirectsFileSupport(t *testing.T) { fixture := car.MustOpenUnixfsCar("t0109-redirects.car") - redirectDir := fixture.MustGetNode("examples") redirectDirCID := redirectDir.Base32Cid() @@ -52,6 +53,7 @@ func TestRedirectsFileSupport(t *testing.T) { Name: "request for $REDIRECTS_DIR_HOSTNAME/redirect-one redirects with default of 301, per _redirects file", Request: Request(). DoNotFollowRedirects(). + Header("Host", u.Host). URL("%s/redirect-one", redirectDirBaseURL), Response: Expect(). Status(301). @@ -234,18 +236,18 @@ func TestRedirectsFileSupport(t *testing.T) { // test_should_not_contain "my 404" response // // ' - { - // TODO: how to test this correctly? - Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", - Hint: ` - We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. - `, - Request: Request(). - URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), - Response: Expect(). - Status(404). - Body(Not(Contains(custom404.ReadFile()))), - }, + // { + // // TODO: confirm with lidel: this should be skipped + // Name: "This test ensures _redirects is supported only on Web Gateways that use Host header (DNSLink, Subdomain)", + // Hint: ` + // We expect the request to fail with a 404 (do not use the _redirect), and that 404 should not contain the custom 404 body. + // `, + // Request: Request(). + // URL("http://127.0.0.1:8080/ipfs/%s/301-redirect-one", redirectDirCID), + // Response: Expect(). + // Status(404). + // Body(Not(Contains(custom404.ReadFile()))), + // }, }...) // # Invalid file, containing forced redirect @@ -304,88 +306,102 @@ func TestRedirectsFileSupport(t *testing.T) { }...) } - if specs.SubdomainGateway.IsEnabled() { - Run(t, unwrapTests(t, tests)) - } else { - t.Skip("subdomain gateway disabled") - } -} - -// TODO: dnslink tests - -func unwrapTests(t *testing.T, tests SugarTests) SugarTests { - t.Helper() - - var out SugarTests - for _, test := range tests { - out = append(out, unwrapTestForGateway(t, test)...) - } - return out + RunIfSpecsAreEnabled( + t, + helpers.UnwrapSubdomainTests(t, tests), + specs.SubdomainGateway, + ) } -func unwrapTestForGateway(t *testing.T, test SugarTest) SugarTests { - t.Helper() +func TestRedirectsFileSupportWithDNSLink(t *testing.T) { + dnsLinks := dnslink.MustOpenDNSLink("t0109-dnslink.yml") + dnsLink := dnsLinks.MustGet("custom-dnslink") - baseURL := test.Request.GetURL() - req := test.Request - expected := test.Response - - u, err := url.Parse(baseURL) + gatewayURL := SubdomainGatewayURL + u, err := url.Parse(gatewayURL) if err != nil { t.Fatal(err) } - // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed - // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is - // actually living on http://127.0.0.1:8080 or somewhere else. - // - // The test knows two addresses: - // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. - // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. - // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` - host := u.Host + dnsLinkBaseUrl := fmt.Sprintf("%s://%s.%s", u.Scheme, dnsLink, u.Host) - // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` - u.Host = GatewayHost - rawURL := u.String() + tests := SugarTests{ + // # make sure test setup is valid (fail if CoreAPI is unable to resolve) + // test_expect_success "spoofed DNSLink record resolves in cli" " + // ipfs resolve /ipns/$DNSLINK_FQDN > result && + // test_should_contain \"$REDIRECTS_DIR_CID\" result && + // ipfs cat /ipns/$DNSLINK_FQDN/_redirects > result && + // test_should_contain \"index.html\" result + // " + // SKIPPED - return SugarTests{ + // test_expect_success "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file" ' + // curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && + // test_should_contain "301 Moved Permanently" response && + // test_should_contain "Location: /one.html" response + // ' { - Name: fmt.Sprintf("%s (direct HTTP)", test.Name), - Hint: fmt.Sprintf("%s\n%s", test.Hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), - Request: req. - URL(rawURL). - DoNotFollowRedirects(). + Name: "request for $DNSLINK_FQDN/redirect-one redirects with default of 301, per _redirects file", + Request: Request(). + URL("%s/redirect-one", dnsLinkBaseUrl), + Response: Expect(). + Status(301). Headers( - Header("Host", host), + Header("Location", "/one.html"), ), - Response: expected, - }, - { - Name: fmt.Sprintf("%s (HTTP proxy)", test.Name), - Hint: fmt.Sprintf("%s\n%s", test.Hint, "HTTP proxy (hostname is passed via URL)"), - Request: req. - URL(baseURL). - Proxy(GatewayURL). - DoNotFollowRedirects(), - Response: expected, }, + // # ensure custom 404 works and has the same cache headers as regular /ipns/ paths + // test_expect_success "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file" ' + // curl -sD - --resolve $DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$DNSLINK_FQDN:$GWAY_PORT/not-found/has-no-redirects-entry" > response && + // test_should_contain "404 Not Found" response && + // test_should_contain "Etag: \"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\"" response && + // test_should_not_contain "Cache-Control: public, max-age=29030400, immutable" response && + // test_should_not_contain "immutable" response && + // test_should_contain "Date: " response && + // test_should_contain "my 404" response + // ' { - Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", test.Name), - Hint: fmt.Sprintf("%s\n%s", test.Hint, `HTTP proxy - In HTTP/1.x, the pseudo-method CONNECT, - can be used to convert an HTTP connection into a tunnel to a remote host - https://tools.ietf.org/html/rfc7231#section-4.3.6 - `), - Request: req. - URL(baseURL). - Proxy(GatewayURL). - WithProxyTunnel(). - DoNotFollowRedirects(). + Name: "request for $DNSLINK_FQDN/en/has-no-redirects-entry returns custom 404, per _redirects file", + Hint: `ensure custom 404 works and has the same cache headers as regular /ipns/ paths`, + Request: Request(). + URL("%s/not-found/has-no-redirects-entry", dnsLinkBaseUrl), + Response: Expect(). + Status(404). Headers( - Header("Host", host), + Header("Etag", "\"Qmd9GD7Bauh6N2ZLfNnYS3b7QVAijbud83b8GE8LPMNBBP\""), + Header("Cache-Control").Not().Contains("public, max-age=29030400, immutable"), + Header("Cache-Control").Not().Contains("immutable"), + Header("Date").Exists(), + ). + Body( + // TODO: I like the readable part here, maybe rewrite to load the file. + Contains("my 404"), ), - Response: expected, }, + // test_expect_success "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled" ' + // curl -sD - --resolve $NO_DNSLINK_FQDN:$GWAY_PORT:127.0.0.1 "http://$NO_DNSLINK_FQDN:$GWAY_PORT/redirect-one" > response && + // test_should_not_contain "one.html" response && + // test_should_not_contain "301 Moved Permanently" response && + // test_should_not_contain "Location:" response + // ' + // TODO(lidel): this test seems to validate some kubo behavior not really gateway. + // { + // Name: "request for $NO_DNSLINK_FQDN/redirect-one does not redirect, since DNSLink is disabled", + // Request: Request(). + // URL("%s/redirect-one", noDnsLinkBaseUrl), + // Response: Expect(). + // // TODO: add "status not equal to 301" check. + // // TODO: what `test_should_not_contain "one.html" response` actually means? No location correct? + // Headers( + // Header("Location").Not().Exists(), + // ), + // }, } + + RunIfSpecsAreEnabled( + t, + helpers.UnwrapSubdomainTests(t, tests), + specs.DNSLinkResolver, + specs.SubdomainGateway, + ) } diff --git a/tooling/dnslink/dnslink.go b/tooling/dnslink/dnslink.go new file mode 100644 index 000000000..2a8778ab9 --- /dev/null +++ b/tooling/dnslink/dnslink.go @@ -0,0 +1,52 @@ +package dnslink + +import ( + "fmt" + "os" + "path" + + "github.com/ipfs/gateway-conformance/tooling/fixtures" + "gopkg.in/yaml.v3" +) + +type DNSLinks struct { + DNSLinks map[string]DNSLink `yaml:"dnslinks"` +} + +type DNSLink struct { + Subdomain string `yaml:"subdomain"` + Path string `yaml:"path"` +} + +func OpenDNSLink(absPath string) (*DNSLinks, error) { + data, err := os.ReadFile(absPath) + if err != nil { + return nil, err + } + + var dnsLinks DNSLinks + err = yaml.Unmarshal(data, &dnsLinks) + if err != nil { + return nil, err + } + + return &dnsLinks, nil +} + +func MustOpenDNSLink(file string) *DNSLinks { + fixturePath := path.Join(fixtures.Dir(), file) + dnsLinks, err := OpenDNSLink(fixturePath) + if err != nil { + panic(err) + } + + return dnsLinks +} + +func (d *DNSLinks) MustGet(id string) string { + dnsLink, ok := d.DNSLinks[id] + if !ok { + panic(fmt.Errorf("dnslink %s not found", id)) + } + return dnsLink.Subdomain +} diff --git a/tooling/dnslink/merge.go b/tooling/dnslink/merge.go new file mode 100644 index 000000000..2d7a91356 --- /dev/null +++ b/tooling/dnslink/merge.go @@ -0,0 +1,43 @@ +package dnslink + +import ( + "encoding/json" + "fmt" + "os" +) + +func Aggregate(inputPaths []string) (map[string]string, error) { + aggMap := make(map[string]string) + + for _, file := range inputPaths { + dnsLinks, err := OpenDNSLink(file) + if err != nil { + return nil, fmt.Errorf("error loading file %s: %v", file, err) + } + + for _, link := range dnsLinks.DNSLinks { + if _, ok := aggMap[link.Subdomain]; ok { + return nil, fmt.Errorf("collision detected for subdomain %s", link.Subdomain) + } + + aggMap[link.Subdomain] = link.Path + } + } + + return aggMap, nil +} + +func Merge(inputPaths []string, outputPath string) error { + kvs, err := Aggregate(inputPaths) + if err != nil { + return err + } + + j, err := json.MarshalIndent(kvs, "", " ") + if err != nil { + return err + } + + err = os.WriteFile(outputPath, j, 0644) + return err +} \ No newline at end of file diff --git a/tooling/fixtures/list.go b/tooling/fixtures/list.go index d58cbec83..4151ff2d8 100644 --- a/tooling/fixtures/list.go +++ b/tooling/fixtures/list.go @@ -13,8 +13,14 @@ func Dir() string { return path.Join(home, "fixtures") } -func List() ([]string, error) { +type Fixtures struct { + CarFiles []string + ConfigFiles []string +} + +func List() (*Fixtures, error) { var carFiles []string + var yamlFiles []string err := filepath.WalkDir(Dir(), func(path string, d os.DirEntry, err error) error { if err != nil { @@ -31,6 +37,14 @@ func List() ([]string, error) { carFiles = append(carFiles, path) } + // if we have a yaml file, append: + if filepath.Ext(path) == ".yml" { + path, err := filepath.Abs(path) + if err != nil { + return err + } + yamlFiles = append(yamlFiles, path) + } return nil }) @@ -39,5 +53,8 @@ func List() ([]string, error) { return nil, err } - return carFiles, nil + return &Fixtures{ + CarFiles: carFiles, + ConfigFiles: yamlFiles, + }, nil } diff --git a/tooling/helpers/subdomain.go b/tooling/helpers/subdomain.go new file mode 100644 index 000000000..7049d3e3e --- /dev/null +++ b/tooling/helpers/subdomain.go @@ -0,0 +1,90 @@ +package helpers + +import ( + "fmt" + "net/url" + "testing" + + "github.com/ipfs/gateway-conformance/tooling/test" +) + +/** + * UnwrapSubdomainTests takes a list of tests and returns a (larger) list of tests + * that will run on the subdomain gateway. + */ +func UnwrapSubdomainTests(t *testing.T, tests test.SugarTests) test.SugarTests { + t.Helper() + + var out test.SugarTests + for _, test := range tests { + out = append(out, unwrapSubdomainTest(t, test)...) + } + return out +} + +func unwrapSubdomainTest(t *testing.T, unwraped test.SugarTest) test.SugarTests { + t.Helper() + + baseURL := unwraped.Request.GetURL() + req := unwraped.Request + expected := unwraped.Response + + u, err := url.Parse(baseURL) + if err != nil { + t.Fatal(err) + } + // Because you might be testing an IPFS node in CI, or on your local machine, the test are designed + // to test the subdomain behavior (querying http://{CID}.my-subdomain-gateway.io/) even if the node is + // actually living on http://127.0.0.1:8080 or somewhere else. + // + // The test knows two addresses: + // - GatewayURL: the URL we connect to, it might be "dweb.link", "127.0.0.1:8080", etc. + // - SubdomainGatewayURL: the URL we test for subdomain requests, it might be "dweb.link", "localhost", "example.com", etc. + + // host is the hostname of the gateway we are testing, it might be `localhost` or `example.com` + host := u.Host + + // raw url is the url but we replace the host with our local url, it might be `http://127.0.0.1/ipfs/something` + u.Host = test.GatewayHost + rawURL := u.String() + + return test.SugarTests{ + { + Name: fmt.Sprintf("%s (direct HTTP)", unwraped.Name), + Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, "direct HTTP request (hostname in URL, raw IP in Host header)"), + Request: req. + URL(rawURL). + DoNotFollowRedirects(). + Headers( + test.Header("Host", host), + ), + Response: expected, + }, + { + Name: fmt.Sprintf("%s (HTTP proxy)", unwraped.Name), + Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, "HTTP proxy (hostname is passed via URL)"), + Request: req. + URL(baseURL). + Proxy(test.GatewayURL). + DoNotFollowRedirects(), + Response: expected, + }, + { + Name: fmt.Sprintf("%s (HTTP proxy tunneling via CONNECT)", unwraped.Name), + Hint: fmt.Sprintf("%s\n%s", unwraped.Hint, `HTTP proxy + In HTTP/1.x, the pseudo-method CONNECT, + can be used to convert an HTTP connection into a tunnel to a remote host + https://tools.ietf.org/html/rfc7231#section-4.3.6 + `), + Request: req. + URL(baseURL). + Proxy(test.GatewayURL). + WithProxyTunnel(). + DoNotFollowRedirects(). + Headers( + test.Header("Host", host), + ), + Response: expected, + }, + } +} diff --git a/tooling/specs/specs.go b/tooling/specs/specs.go index a44ff42b5..cc009d9d9 100644 --- a/tooling/specs/specs.go +++ b/tooling/specs/specs.go @@ -1,6 +1,8 @@ package specs -import "fmt" +import ( + "fmt" +) type maturity string @@ -26,11 +28,13 @@ type Spec string const ( SubdomainGateway Spec = "subdomain-gateway" + DNSLinkResolver Spec = "dnslink-resolver" ) // All specs should be listed here. var specMaturity = map[Spec]maturity{ SubdomainGateway: stable, + DNSLinkResolver: stable, } func (s Spec) IsMature() bool { diff --git a/tooling/test/test.go b/tooling/test/test.go index 388194bc3..5b62e2994 100644 --- a/tooling/test/test.go +++ b/tooling/test/test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/ipfs/gateway-conformance/tooling/check" + "github.com/ipfs/gateway-conformance/tooling/specs" ) type SugarTest struct { @@ -19,9 +20,27 @@ type SugarTest struct { type SugarTests []SugarTest -func Run(t *testing.T, tests SugarTests) { - // NewDialer() +func RunIfSpecsAreEnabled( + t *testing.T, + tests SugarTests, + required ...specs.Spec, +) { + missing := []specs.Spec{} + for _, spec := range required { + if !spec.IsEnabled() { + missing = append(missing, spec) + } + } + + if len(missing) > 0 { + t.Skipf("skipping tests, missing specs: %v", missing) + return + } + Run(t, tests) +} + +func Run(t *testing.T, tests SugarTests) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { method := test.Request.Method_