feat: finish porting and cleaning all simple tests

CHANGELOG.md

@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - `--verbose` flag displays all the output to the console
 
+### Changed
+- finalized port of Kubo's sharness tests. [PR](https://github.com/ipfs/gateway-conformance/pull/92)
+
 ## [0.2.0] - 2023-06-26
 ### Added
 - `carFixture.MustGetChildren`

fixtures/README.md

@@ -1,33 +1,22 @@
-# ipfs add --recursive --raw-leaves --cid-version=1 --wrap-with-directory fixtures/dir
+# Fixtures
 
-added bafkreibyniihaqnhs536mbvc3wx66vbfttisc6zrvea6kzeku3w4as6lum dir/ascii.txt
-added bafybeiaxmpwa4nz2mekiiosayvkfgfpbubm7pch7tspx5syhzzsvkoewc4 dir
-added bafybeie7u7bbgvdqbur6inl6fpl4yw3xcg34bjjmnnzkmpwb5v6x3xz3hy
+This folder contains all the fixtures using during the tests, with
+the recipes to re-create them.
 
-# ipfs dag export bafybeie7u7bbgvdqbur6inl6fpl4yw3xcg34bjjmnnzkmpwb5v6x3xz3hy > fixtures/dir.car
+## Recipes
 
-# go install github.com/ipld/go-car/cmd/car@latest
+### [gateway-raw-block.car](./gateway-raw-block.car)
 
-# car list fixtures/dir.car
+Generated with:
 
-bafybeie7u7bbgvdqbur6inl6fpl4yw3xcg34bjjmnnzkmpwb5v6x3xz3hy
-bafybeiaxmpwa4nz2mekiiosayvkfgfpbubm7pch7tspx5syhzzsvkoewc4
-bafkreibyniihaqnhs536mbvc3wx66vbfttisc6zrvea6kzeku3w4as6lum
+```sh
+# using Kubo CLI version 0.18.1 (https://dist.ipfs.tech/kubo/v0.18.1/)
+mkdir -p dir &&
+echo "hello application/vnd.ipld.raw" > dir/ascii.txt &&
+ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) &&
+FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/dir/ascii.txt | cut -d "/" -f3) &&
+ipfs dag export $ROOT_DIR_CID > gateway-raw-block.car
 
-# car extract --file fixtures/dir.car --verbose
-
-.../gateway-conformance/dir
-.../gateway-conformance/dir/ascii.txt
-
-# go run extract.go fixtures/dir.car
-
-path: /
-  cid: bafybeie7u7bbgvdqbur6inl6fpl4yw3xcg34bjjmnnzkmpwb5v6x3xz3hy
-  raw: [18 45 10 36 1 112 18 32 23 99 236 14 55 58 97 20 132 58 64 197 84 83 21 225 160 89 247 136 255 156 159 126 203 7 206 101 85 56 150 23 18 3 100 105 114 24 90 10 2 8 1]
-path: /dir
-  cid: bafybeiaxmpwa4nz2mekiiosayvkfgfpbubm7pch7tspx5syhzzsvkoewc4
-  raw: [18 51 10 36 1 85 18 32 56 106 16 112 65 167 151 119 230 6 162 221 175 239 84 37 156 209 33 123 49 169 1 229 100 138 166 237 192 75 203 163 18 9 97 115 99 105 105 46 116 120 116 24 33 10 2 8 1]
-path: /dir/ascii.txt
-  cid: bafkreibyniihaqnhs536mbvc3wx66vbfttisc6zrvea6kzeku3w4as6lum
-  raw: [103 111 111 100 98 121 101 32 97 112 112 108 105 99 97 116 105 111 110 47 118 110 100 46 105 112 108 100 46 114 97 119 10]
-  str: goodbye application/vnd.ipld.raw
+echo ROOT_DIR_CID=${ROOT_DIR_CID} # ./
+echo FILE_CID=${FILE_CID} # ./dir/ascii.txt
+```

fixtures/t0115/README.md → fixtures/dir_listing/README.md

@@ -1,12 +1,13 @@
-# Dataset description/sources
+# Directory Listing Fixtures
 
-- fixtures.car
-  - raw CARv1
+## Recipes
 
-generated with:
+### [fixtures.car](./fixtures.car)
+
+Raw CARv1, generated with:
 
 ```sh
-# using ipfs version 0.18.1
+# using Kubo CLI version 0.18.1 (https://dist.ipfs.tech/kubo/v0.18.1/)
 mkdir -p rootDir/ipfs &&
 mkdir -p rootDir/ipns &&
 mkdir -p rootDir/api &&

fixtures/gateway-cache/README.md

@@ -10,7 +10,7 @@
 generated with:
 
 ```sh
-# using ipfs version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867)
+# using Kubo CLI version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867)
 
 mkdir -p root2/root3/root4 &&
 echo "hello" > root2/root3/root4/index.html &&

fixtures/ipns_records/README.md

@@ -0,0 +1,23 @@
+# IPNS Records Fixtures
+
+## Recipes
+
+### Fixtures and ipns-record
+
+```sh
+# using Kubo CLI version 0.21.0-rc3 (https://dist.ipfs.tech/kubo/v0.21.0-rc3/)
+FILE_CID=$(echo "Hello IPFS" | ipfs add --cid-version 1 -q)
+IPNS_KEY=$(ipfs key gen ipns-record)
+
+ipfs dag export ${FILE_CID} > fixtures.car
+
+# publish a record valid for a 100 years
+ipfs name publish --key=ipns-record --quieter --ttl=876600h --lifetime=876600h /ipfs/${FILE_CID}
+ipfs routing get /ipns/${IPNS_KEY} > ${IPNS_KEY}.ipns-record
+
+echo IPNS_KEY=${IPNS_KEY}
+echo FILE_CID=${FILE_CID} # A file containing "Hello IPFS"
+
+# IPNS_KEY=k51qzi5uqu5dh71qgwangrt6r0nd4094i88nsady6qgd1dhjcyfsaqmpp143ab
+# FILE_CID=bafkreidfdrlkeq4m4xnxuyx6iae76fdm4wgl5d4xzsb77ixhyqwumhz244 # A file containing Hello IPFS
+```

fixtures/path_gateway_dag/README.md

@@ -0,0 +1,66 @@
+# Path Gateway Dag Fixtures
+
+## Recipes
+
+### Legacy fixtures from Kubo
+
+- dag-cbor-traversal.car
+- dag-json-traversal.car
+- dag-pb.car
+- dag-pb.json
+
+### [gateway-json-cbor.car](./gateway-json-cbor.car) and ipns-records
+
+Generated with:
+
+```sh
+# using Kubo CLI version 0.21.0-rc3 (https://dist.ipfs.tech/kubo/v0.21.0-rc3/)
+
+mkdir -p rootDir/ipfs &&
+mkdir -p rootDir/ipns &&
+mkdir -p rootDir/api &&
+mkdir -p rootDir/ą/ę &&
+echo "{ \"test\": \"i am a plain json file\" }" > rootDir/ą/ę/t.json &&
+echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
+echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
+echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
+echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
+DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
+FILE_JSON_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/t.json | jq -r .Hash) &&
+FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
+FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
+echo "$FILE_CID / $FILE_SIZE"
+
+echo DIR_CID=${DIR_CID} # ./rootDir
+echo FILE_JSON_CID=${FILE_JSON_CID} # ./rootDir/ą/ę/t.json
+echo FILE_CID=${FILE_CID} # ./rootDir/ą/ę/file-źł.txt
+echo FILE_SIZE=${FILE_SIZE}
+
+ipfs dag export ${DIR_CID} > gateway-json-cbor.car
+
+DAG_CBOR_TRAVERSAL_CID="bafyreibs4utpgbn7uqegmd2goqz4bkyflre2ek2iwv743fhvylwi4zeeim"
+DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxkd47xq"
+DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke"
+
+test_native_dag() {
+  NAME=$1
+  CID=$2
+
+  IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 ${NAME}_test_key | head -n1 | tr -d "\n")
+  ipfs name publish --key ${NAME}_test_key --allow-offline --ttl=876600h --lifetime=876600h -Q "/ipfs/${CID}" > name_publish_out
+
+  ipfs routing get /ipns/${IPNS_ID} > ${IPNS_ID}.ipns-record
+
+  echo "IPNS_ID_${NAME}=${IPNS_ID}"
+}
+
+test_native_dag "DAG_JSON" "$DAG_JSON_TRAVERSAL_CID"
+test_native_dag "DAG_CBOR" "$DAG_CBOR_TRAVERSAL_CID"
+
+# DIR_CID=bafybeiafyvqlazbbbtjnn6how5d6h6l6rxbqc4qgpbmteaiskjrffmyy4a # ./rootDir
+# FILE_JSON_CID=bafkreibrppizs3g7axs2jdlnjua6vgpmltv7k72l7v7sa6mmht6mne3qqe # ./rootDir/ą/ę/t.json
+# FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
+# FILE_SIZE=34
+# IPNS_ID_DAG_JSON=k51qzi5uqu5dhjghbwdvbo6mi40htrq6e2z4pwgp15pgv3ho1azvidttzh8yy2
+# IPNS_ID_DAG_CBOR=k51qzi5uqu5dghjous0agrwavl8vzl64xckoqzwqeqwudfr74kfd11zcyk3b7l
+```

fixtures/path_gateway_tar/README.md

@@ -0,0 +1,40 @@
+# Path Gateway Tar Fixtures
+
+## Recipes
+
+### [inside-roots.car](./inside-root.car) and [outside-roots.car](./outside-root.car)
+
+Legacy fixtures from Kubo Sharness.
+
+See the [PR](https://github.com/ipfs/kubo/pull/9029) that introduced them for more details
+
+### [fixtures.car](./fixtures.car)
+
+Generated with:
+
+```sh
+# Kubo CLI version 0.18.1 (https://dist.ipfs.tech/kubo/v0.18.1/)
+
+mkdir -p rootDir/ipfs &&
+mkdir -p rootDir/ipns &&
+mkdir -p rootDir/api &&
+mkdir -p rootDir/ą/ę &&
+echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
+echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
+echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
+echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
+DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
+FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
+FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
+echo "$FILE_CID / $FILE_SIZE"
+
+echo DIR_CID=${DIR_CID} # ./rootDir
+echo FILE_CID=${FILE_CID} # ./rootDir/ą/ę/file-źł.txt
+echo FILE_SIZE=${FILE_SIZE}
+
+ipfs dag export ${DIR_CID} > ./fixtures.car
+
+# DIR_CID=bafybeig6ka5mlwkl4subqhaiatalkcleo4jgnr3hqwvpmsqfca27cijp3i # ./rootDir
+# FILE_CID=bafkreialihlqnf5uwo4byh4n3cmwlntwqzxxs2fg5vanqdi3d7tb2l5xkm # ./rootDir/ą/ę/file-źł.txt
+# FILE_SIZE=34
+```

fixtures/path_gateway_unixfs/README.md

@@ -0,0 +1,14 @@
+# Path Gateway Fixtures
+
+## Recipes
+
+### [symlink.car](./symlink.car)
+
+```sh
+# using Kubo CLI version 0.18.1 (https://dist.ipfs.tech/kubo/v0.18.1/
+mkdir testfiles &&
+echo "content" > testfiles/foo &&
+ln -s foo testfiles/bar &&
+ROOT_DIR_CID=$(ipfs add -Qr testfiles) &&
+ipfs dag export $ROOT_DIR_CID > symlink.car
+```

fixtures/redirects_file/README.md

@@ -0,0 +1,12 @@
+# Fixture for Redirects File Testing
+
+## Recipes
+
+### [dnslink.yml](./dnslink.yml)
+
+See comments in the yml file.
+
+
+### [redirects.car](./redirects.car)
+
+Fixtures based on [specs.ipfs.tech/http-gateways/web-redirects-file/#test-fixtures](https://specs.ipfs.tech/http-gateways/web-redirects-file/#test-fixtures) and [IPIP-0002](https://specs.ipfs.tech/ipips/ipip-0002/)

fixtures/t0109-dnslink.yml → fixtures/redirects_file/dnslink.yml

@@ -3,5 +3,5 @@ dnslinks:
   custom-dnslink:
     subdomain: dnslink-enabled-on-fqdn
     # this is the cid of the folder
-    # t0109-redirects.car:/examples/
+    # ./redirects.car:/examples/
     path: /ipfs/QmYBhLYDwVFvxos9h8CGU2ibaY66QNgv8hpfewxaQrPiZj

fixtures/t0114/README.md → fixtures/subdomain_gateway/README.md

@@ -1,18 +1,15 @@
-# Dataset description/sources
+# Subdomain Gateway Fixtures
 
-- fixtures.car
-  - raw CARv1
+## Recipes
 
-- QmUKd....ipns-record
-  - ipns record, encoded with protocol buffer
+### [dnslink.yml](./dnslink.yml)
 
-- 12D3K....ipns-record
-  - ipns record, encoded with protocol buffer
+See comments in the yml file.
 
-Generated with:
+### [fixtures.car](./fixtures.car)
 
 ```sh
-# using ipfs version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867)
+# using Kubo CLI version 0.21.0-rc3 (https://dist.ipfs.tech/kubo/v0.21.0-rc3/)
 
 # CIDv0to1 is necessary because raw-leaves are enabled by default during
 # "ipfs add" with CIDv1 and disabled with CIDv0
@@ -42,18 +39,22 @@ DIR_CID=$(ipfs add -Qr --cid-version 1 testdirlisting)
 echo DIR_CID=${DIR_CID} # ./testdirlisting
 
 ipfs files mkdir /t0114/
-ipfs files cp /ipfs/${CIDv1} /t0114/
-ipfs files cp /ipfs/${CIDv0} /t0114/
-ipfs files cp /ipfs/${CIDv0to1} /t0114/
-ipfs files cp /ipfs/${DIR_CID} /t0114/
-ipfs files cp /ipfs/${CIDv1_TOO_LONG} /t0114/
+ipfs files cp /ipfs/${CIDv1} /t0114/hello-CIDv1
+ipfs files cp /ipfs/${CIDv0} /t0114/hello-CIDv0
+ipfs files cp /ipfs/${CIDv0to1} /t0114/hello-CIDv0to1
+ipfs files cp /ipfs/${DIR_CID} /t0114/testdirlisting
+ipfs files cp /ipfs/${CIDv1_TOO_LONG} /t0114/hello-CIDv1_TOO_LONG
 
 ROOT=`ipfs files stat /t0114/ --hash`
 
 ipfs dag export ${ROOT} > ./fixtures.car
+```
+
+### [ipns-records](./)
 
-# Then the keys
+We create ipns-records pointing to the CIDv1 generated above:
 
+```sh
 KEY_NAME=test_key_rsa_$RANDOM
 RSA_KEY=$(ipfs key gen --ipns-base=b58mh --type=rsa --size=2048 ${KEY_NAME} | head -n1 | tr -d "\n")
 RSA_IPNS_IDv0=$(echo "$RSA_KEY" | ipfs cid format -v 0)
@@ -89,23 +90,4 @@ echo ED25519_IPNS_IDv1=${ED25519_IPNS_IDv1}
 echo ED25519_IPNS_IDv1_DAGPB=${ED25519_IPNS_IDv1_DAGPB}
 echo IPNS_ED25519_B58MH=${IPNS_ED25519_B58MH}
 echo IPNS_ED25519_B36CID=${IPNS_ED25519_B36CID}
-
-# CID_VAL=hello
-# CIDv1=bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
-# CIDv0=QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN
-# CIDv0to1=bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4
-# CIDv1_TOO_LONG=bafkrgqhhyivzstcz3hhswshfjgy6ertgmnqeleynhwt4dlfsthi4hn7zgh4uvlsb5xncykzapi3ocd4lzogukir6ksdy6wzrnz6ohnv4aglcs
-# DIR_CID=bafybeiht6dtwk3les7vqm6ibpvz6qpohidvlshsfyr7l5mpysdw2vmbbhe # ./testdirlisting
-
-# RSA_KEY=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3
-# RSA_IPNS_IDv0=QmVujd5Vb7moysJj8itnGufN7MEtPRCNHkKpNuA4onsRa3
-# RSA_IPNS_IDv1=k2k4r8m7xvggw5pxxk3abrkwyer625hg01hfyggrai7lk1m63fuihi7w
-# RSA_IPNS_IDv1_DAGPB=k2jmtxu61bnhrtj301lw7zizknztocdbeqhxgv76l2q9t36fn9jbzipo
-
-# ED25519_KEY=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
-# ED25519_IPNS_IDv0=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
-# ED25519_IPNS_IDv1=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam
-# ED25519_IPNS_IDv1_DAGPB=k50rm9yjlt0jey4fqg6wafvqprktgbkpgkqdg27tpqje6iimzxewnhvtin9hhq
-# IPNS_ED25519_B58MH=12D3KooWLQzUv2FHWGVPXTXSZpdHs7oHbXub2G5WC8Tx4NQhyd2d
-# IPNS_ED25519_B36CID=k51qzi5uqu5dk3v4rmjber23h16xnr23bsggmqqil9z2gduiis5se8dht36dam
 ```

fixtures/t0123/README.md

@@ -14,7 +14,7 @@
 generated with:
 
 ```sh
-# using ipfs version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867)
+# using Kubo CLI version 0.21.0-dev (03a98280e3e642774776cd3d0435ab53e5dfa867)
 
 mkdir -p rootDir/ipfs &&
 mkdir -p rootDir/ipns &&