diff --git a/go.mod b/go.mod index 171a3611..5c6e6cbd 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipld-format v0.5.0 - github.com/ipfs/go-libipfs v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-unixfsnode v1.7.1 github.com/ipld/go-car/v2 v2.10.1 @@ -37,6 +36,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.24.4 + github.com/warpfork/go-testmark v0.12.1 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/multierr v1.11.0 @@ -84,6 +84,7 @@ require ( github.com/ipfs/go-bitfield v1.1.0 // indirect github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect + github.com/ipfs/go-ipfs-files v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-cbor v0.0.6 // indirect diff --git a/go.sum b/go.sum index c2e72b7c..65562672 100644 --- a/go.sum +++ b/go.sum @@ -293,10 +293,10 @@ github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= +github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= -github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= @@ -312,7 +312,6 @@ github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAF github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= -github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= @@ -614,7 +613,9 @@ github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/warpfork/go-fsx v0.3.0/go.mod h1:oTACCMj+Zle+vgVa5SAhGAh7WksYpLgGUCKEAVc+xPg= github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= +github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= diff --git a/pkg/internal/fixtures/testcases.go b/pkg/internal/fixtures/testcases.go new file mode 100644 index 00000000..896eb897 --- /dev/null +++ b/pkg/internal/fixtures/testcases.go @@ -0,0 +1,94 @@ +package fixtures + +import ( + "errors" + "fmt" + "net/url" + "strings" + + "github.com/filecoin-project/lassie/pkg/types" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" +) + +type TestCase struct { + Name string + Root cid.Cid + Path string + Scope types.DagScope + Duplicates bool + ByteRange *types.ByteRange + ExpectedCids []cid.Cid +} + +func (tc TestCase) AsQuery() string { + pp := ipld.ParsePath(tc.Path).String() + if pp != "" { + pp = "/" + pp + } + br := "" + if tc.ByteRange != nil && !tc.ByteRange.IsDefault() { + br = fmt.Sprintf("&entity-bytes=%s", tc.ByteRange.String()) + } + dup := "" + if tc.Duplicates { + dup = "&dups=y" + } + return fmt.Sprintf("/ipfs/%s%s?dag-scope=%s%s%s", tc.Root, pp, tc.Scope, br, dup) +} + +func ParseCase(name, spec, exec string) (TestCase, error) { + lines := strings.Split(exec, "\n") + for len(lines) > 0 && strings.TrimSpace(lines[0]) == "" { + lines = lines[1:] + } + for len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" { + lines = lines[:len(lines)-1] + } + specParts := strings.Split(strings.TrimSpace(spec), "?") + if len(specParts) != 2 { + return TestCase{}, errors.New("invalid spec") + } + spec = specParts[0] + query, err := url.ParseQuery(specParts[1]) + if err != nil { + return TestCase{}, err + } + specParts = strings.Split(spec, "/") + if specParts[0] != "" && specParts[1] != "ipfs" { + return TestCase{}, errors.New("invalid spec") + } + root, err := cid.Parse(specParts[2]) + if err != nil { + return TestCase{}, err + } + path := "/" + ipld.ParsePath(strings.Join(specParts[3:], "/")).String() + scope, err := types.ParseDagScope(query.Get("dag-scope")) // required + if err != nil { + return TestCase{}, err + } + duplicates := query.Get("dups") == "y" + var byteRange *types.ByteRange + if query.Get("byte-range") != "" { + if br, err := types.ParseByteRange(query.Get("byte-range")); err != nil { + return TestCase{}, err + } else { + byteRange = &br + } + } + expectedCids := make([]cid.Cid, 0, len(lines)) + for _, line := range lines { + la := strings.Split(line, "|") + c := cid.MustParse(strings.TrimSpace(la[0])) + expectedCids = append(expectedCids, c) + } + return TestCase{ + Name: name, + Root: root, + Path: path, + Scope: scope, + Duplicates: duplicates, + ByteRange: byteRange, + ExpectedCids: expectedCids, + }, nil +} diff --git a/pkg/internal/fixtures/unixfs_20m_variety.go b/pkg/internal/fixtures/unixfs_20m_variety.go new file mode 100644 index 00000000..8937842a --- /dev/null +++ b/pkg/internal/fixtures/unixfs_20m_variety.go @@ -0,0 +1,75 @@ +package fixtures + +import ( + "fmt" + "io" + "os" + "strings" + + "github.com/ipfs/go-cid" + carstorage "github.com/ipld/go-car/v2/storage" + "github.com/ipld/go-ipld-prime/storage" + "github.com/warpfork/go-testmark" +) + +var Unixfs20mVarietyRoot = cid.MustParse("bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq") + +const file = "internal/testdata/unixfs_20m_variety." + +func filepath(typ string) (string, error) { + wd, err := os.Getwd() + if err != nil { + return "", err + } + rootInd := strings.LastIndex(wd, "/lassie/pkg/") + if rootInd == -1 { + return "", fmt.Errorf("could not find root of lassie package") + } + filename := wd[:rootInd] + "/lassie/pkg/" + file + typ + fmt.Println("Using", filename) + return filename, nil +} + +func Unixfs20mVarietyReadableStorage() (storage.ReadableStorage, io.Closer, error) { + file, err := filepath("car") + if err != nil { + return nil, nil, err + } + carFile, err := os.Open(file) + if err != nil { + return nil, nil, err + } + reader, err := carstorage.OpenReadable(carFile) + if err != nil { + carFile.Close() + return nil, nil, err + } + return reader, carFile, nil +} + +func Unixfs20mVarietyCases() ([]TestCase, error) { + file, err := filepath("md") + if err != nil { + return nil, err + } + doc, err := testmark.ReadFile(file) + if err != nil { + return nil, err + } + doc.BuildDirIndex() + testCases := make([]TestCase, 0) + for _, test := range doc.DirEnt.Children["test"].ChildrenList { + for _, scope := range test.ChildrenList { + tc, err := ParseCase(test.Name+"/"+scope.Name, dstr(scope, "query"), dstr(scope, "execution")) + if err != nil { + return nil, err + } + testCases = append(testCases, tc) + } + } + return testCases, nil +} + +func dstr(dir *testmark.DirEnt, ch string) string { + return string(dir.Children[ch].Hunk.Body) +} diff --git a/pkg/internal/testdata/unixfs_20m_variety.md b/pkg/internal/testdata/unixfs_20m_variety.md new file mode 100644 index 00000000..63bf76cc --- /dev/null +++ b/pkg/internal/testdata/unixfs_20m_variety.md @@ -0,0 +1,364 @@ +# unixfs_20m_variety (test fixture) + + * 20 MB of files with a variety of UnixFS features across 1,103 blocks + * unixfs_20m_variety.car is a CARv1, reverse ordered (build order), without a root + +## Root CID +[testmark]:# (root) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq +``` + +## Test Cases + +### Small file in directory + +Same result regardless of scope. + +#### all + +[testmark]:# (test/small_file_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=all +``` + +[testmark]:# (test/small_file_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +``` + +#### entity + +[testmark]:# (test/small_file_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=entity +``` + +[testmark]:# (test/small_file_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +``` + +#### block + +[testmark]:# (test/small_file_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/pi?dag-scope=block +``` + +[testmark]:# (test/small_file_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafkreigtkfmisjmiqfp2y73lpqj7uu7mnqg7cjm5br67ek6nwsbyuqgkom | RawLeaf | ↳ /pi[0:1701] +``` + +### Sharded file in directory + +All and entity are the same but block should just get the root File block + +#### all + +[testmark]:# (test/sharded_file_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256144] +bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512288] +bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768432] +bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024576] +bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280720] +bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352050] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +bafkreigtwsisgpg6x752y5md2z2r4jhuhohh46y4x3mvxrbkcubo7mqlgi | RawLeaf | ↳ /želva.xml[0:256144] +bafkreigahqispwg55yvqwobavwlheongcyhk63eufsaqutqgjiwfwesfau | RawLeaf | /želva.xml[256144:512288] +bafkreic4kgh44v2ung3wspd7y6wigcxake45ztpfx3c5ibfwmqe2kt7uay | RawLeaf | /želva.xml[512288:768432] +bafkreibdjsvoyftwgcywb3xylnfod2wififs2tj7pww4zkvaba7z5aigtm | RawLeaf | /želva.xml[768432:1024576] +bafkreidtv6frnbb4o4yobyu3xbtd5onzne67dhgngpd3vwrbdcneapy6fa | RawLeaf | /želva.xml[1024576:1280720] +bafkreidy2ntimx4u22b5uy6tjh7du5ex5lhgu7comxwcjfkzfbdsdunbou | RawLeaf | /želva.xml[1280720:1352050] +``` + +#### block + +[testmark]:# (test/sharded_file_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/želva.xml?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeidchppfcvpa644xihhbvbfyiqupsgp4efh4mf3pengqufwkdfrvha | File | ↳ /želva.xml[0:1352050] +``` + +### Sharded file in directory in directory + +Aame as above but one extra level of nesting. + +#### all + +[testmark]:# (test/sharded_file_in_directory_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_directory_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256144] +bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512288] +bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768432] +bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024576] +bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280720] +bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536864] +bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793008] +bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049152] +bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305296] +bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561440] +bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817584] +bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073728] +bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329872] +bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586016] +bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842160] +bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098304] +bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354448] +bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558478] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_directory_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_directory_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +bafkreidz265issniggrhdjbwp3c5o4fnmg7xsa6hbn3kmyo7bsafc3jjsy | RawLeaf | ↳ /eorþscyld.pdf[0:256144] +bafkreiepissrnjsgcanpe6hzen5kv66glcr5x5vw5vhferkjscxdj2uv3y | RawLeaf | /eorþscyld.pdf[256144:512288] +bafkreifohr4rlzjpskbdcntrjzolglq6e5uf7n4mgdiw7wl2yhovhx46ve | RawLeaf | /eorþscyld.pdf[512288:768432] +bafkreihiulihaonenbbtrbptmcmzcg2hvk6dypdxvmposz7ansd6h5mkla | RawLeaf | /eorþscyld.pdf[768432:1024576] +bafkreidbhmgdt3ajzgoeywwxdbdcquone7buyenwhirl6af2z3gmftiys4 | RawLeaf | /eorþscyld.pdf[1024576:1280720] +bafkreib6iawulrjvzet7dxbqdqneqa7kpy2kacvq7fawjvmlgveoakuzzy | RawLeaf | /eorþscyld.pdf[1280720:1536864] +bafkreicbbl7whtfqsoz6tu663qicdisdyvzp4pp4ylggcttc44j63bdhxy | RawLeaf | /eorþscyld.pdf[1536864:1793008] +bafkreifxe2irzoxtzk3ltpunioehodovu4pwzdtjr2lqtmkg7hyh3o5r2e | RawLeaf | /eorþscyld.pdf[1793008:2049152] +bafkreihr2m4zfx4qvpwkp2fnihn7gbjq6gyr2gv5nbibt2nuajnp2k6pvq | RawLeaf | /eorþscyld.pdf[2049152:2305296] +bafkreibhyumtv62kh3d4rvwpxkdlr52uur5thgmbtyuyxas3c424od7cta | RawLeaf | /eorþscyld.pdf[2305296:2561440] +bafkreiatyvemaol2uxceo4suktpmusl53dqgg5bcqi2rkfhlxnnvpigkci | RawLeaf | /eorþscyld.pdf[2561440:2817584] +bafkreias6ygzx2pnowi3hxj5kmed2rm74hgigwijiandz7vamvrohtkpje | RawLeaf | /eorþscyld.pdf[2817584:3073728] +bafkreihf5g2wmsblx67664w2k3m7hj2bg2wnpumlnty7ssw4gj46oj3bxa | RawLeaf | /eorþscyld.pdf[3073728:3329872] +bafkreib7gr6yi6lhl2p5izuxrxpknf5tc5lhkf5tdhzaxw5g66jxqichra | RawLeaf | /eorþscyld.pdf[3329872:3586016] +bafkreigmnnfehab7c4tblwqgydbaoc76y34rkjfetayb6ytql2eywqnq5y | RawLeaf | /eorþscyld.pdf[3586016:3842160] +bafkreiax5uhfktinmfo3ovm7geaf2266u5ya2slq4bob6eeginffbewszu | RawLeaf | /eorþscyld.pdf[3842160:4098304] +bafkreic5chuwrtofd6ymxawcaxulacfewzfi3zroakmkwjkfmadrxrmln4 | RawLeaf | /eorþscyld.pdf[4098304:4354448] +bafkreich7pcb4lnheypmqb4sikk5m6cuqqbxbjzuk767cx47djxac45o64 | RawLeaf | /eorþscyld.pdf[4354448:4558478] +``` + +#### block + +[testmark]:# (test/sharded_file_in_directory_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Flibbertigibbet5/eorþscyld.pdf?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_directory_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeicqlqnzd2tvs5vlo4c2dz72ryvouwmjwpu2mvxrocuj2u26joxeam | Directory | ↳ /Flibbertigibbet5 +bafybeigutoywu5bj3hlcdr4mkm6yepqnwvq5xodulrz2wstwteko2266te | File | ↳ /eorþscyld.pdf[0:4558478] +``` + +### Sharded file in hamt in directory + +Same as above but the inner directory is a HAMT and we have an intermediate block. + +#### all + +[testmark]:# (test/sharded_file_in_hamt_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_hamt_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O +bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256144] +bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512288] +bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768432] +bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024576] +bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280720] +bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536864] +bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793008] +bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049152] +bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305296] +bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561440] +bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817584] +bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977731] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O +bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +bafkreibyblakon7atsv2hwzyzrweeb5uriksuscfxjtlzj3l56xmrouwke | RawLeaf | ↳ /Othello.png[0:256144] +bafkreic7nx6e7zezkdz5lp2xcwarmgavr62cyrmtepyagvm2ivuo3jesqe | RawLeaf | /Othello.png[256144:512288] +bafkreicylwcsi7ovreggex5yc7jolt5zrwf74ida3sprzsw37oxpzkwvte | RawLeaf | /Othello.png[512288:768432] +bafkreiainpsn6gadltlhqzgakxvdbxrz5n2cstizn3afhsr2nwefus7pqm | RawLeaf | /Othello.png[768432:1024576] +bafkreihuwjz4av7scz3fdjejtrx3k6ycdjudjusuz6sjzbbsewpgslosxi | RawLeaf | /Othello.png[1024576:1280720] +bafkreiexmzld3vi72v665bbqn473mfc5smmtd6nnndl7dzcapzlxdwzhq4 | RawLeaf | /Othello.png[1280720:1536864] +bafkreignp6prdhvi4mxoef5ybl4c644eyeikd42w3z2atw5mgbzw5p4374 | RawLeaf | /Othello.png[1536864:1793008] +bafkreia6nxplcal74ycukjwfmwmaaraeg6qwclfvb6qja2iaayukzsedoa | RawLeaf | /Othello.png[1793008:2049152] +bafkreiae6iay4fqiethqcvva4qazdxb7szb4dh77dsxiskvewhm5sepwfm | RawLeaf | /Othello.png[2049152:2305296] +bafkreieprkc6i3qzg47k5y3zxejsoglyl5fwrk3fnslbmdmau7surwiocy | RawLeaf | /Othello.png[2305296:2561440] +bafkreidwsenghahefqcfw5ikxd6osi6a35koyki4gibw3sunlmqrroqzc4 | RawLeaf | /Othello.png[2561440:2817584] +bafkreicn24jr4rwjcptuzu677u2aeqyirf7weradcafcnkq5hzo7wickkq | RawLeaf | /Othello.png[2817584:2977731] +``` + +#### block + +[testmark]:# (test/sharded_file_in_hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/O/Othello.png?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeiahh6giyfzxpsp6b4y6j6r34xjgsb2si7n6dszwoevnjxdcuee5yq | HAMTShard | ↳ /O +bafybeics7ohtet4wfuzszfchbttrgotqzpuruq4bencpdtc443vawbcbni | HAMTShard | ↳ +bafybeigqgljdsq72owvi3mtgjwkrfmmjpevzbya6pu52ufj55h5ypznpx4 | File | ↳ /Othello.png[0:2977731] +``` + +### Sharded file in directory in hamt in directory + +Same as above but with an extra directory layer under the HAMT. + +#### all + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/all/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom/supercalifragilisticexpialidocious.txt?dag-scope=all +``` + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/all/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ +bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom +bafybeify427noacqiu6sxaaunk5uw2xhelnkcwktkvh6woi6ahlipsz7em | File | ↳ /supercalifragilisticexpialidocious.txt[0:568521] +bafkreicdwgfhxwnzq7i34cuhnqqqxz6d6jtirupjy5kq3uaphopyw5e2ky | RawLeaf | ↳ /supercalifragilisticexpialidocious.txt[0:256144] +bafkreifmk6qfl6ucap7btfu35rjd37bh7uuefavq6nsuungxjl3or7bz2e | RawLeaf | /supercalifragilisticexpialidocious.txt[256144:512288] +bafkreifk2vcldftxe57ml2cxyfcwb34ukkhaopm46kv3as5vo26ht63fci | RawLeaf | /supercalifragilisticexpialidocious.txt[512288:568521] +``` + +#### entity + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom?dag-scope=entity +``` + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ +bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom +``` + +#### block + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious/Throttlebottom?dag-scope=block +``` + +[testmark]:# (test/sharded_file_in_directory_in_hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | ↳ +bafybeifjkcls323ddq3t5ov22yl7i6ks36a7jzv33pjy4stqkils6jzvqe | Directory | ↳ /Throttlebottom +``` + +### Hamt in directory + +"All" is too large to be useful, so we'll just do "entity" and "block". + +#### entity + +[testmark]:# (test/hamt_in_directory/entity/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious?dag-scope=entity +``` + +[testmark]:# (test/hamt_in_directory/entity/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +bafybeiggzqpxmfbve7sutrmcwrlwpx4tkno536d7qdgd6hnhxf74t7gzee | HAMTShard | ↳ /Zigzagumptious +bafybeigfwz54atwgnklb72rfqqn2a3ywoontgf3b3kekxwoxalptx7gc3e | HAMTShard | ↳ /Zigzagumptious +bafybeifr3w3eyq2oi6hxwbfoblor7s5pfvu3m4xyyzovxlk4g5xgslsqui | HAMTShard | /Zigzagumptious +bafybeigivdbrxmcixaqfap3rrs7ti5ycr7vj4xfbkjougq67k7ac5oz55a | HAMTShard | /Zigzagumptious +bafybeid3cny2p7fwt4znzm3wjncsesdk2rdze5fm5rzxjpbethy4657j6m | HAMTShard | /Zigzagumptious +bafybeidivwpxv37o6p4ton3xcpmg64pwk4zqlkuge7isafgsgjsvb6ce2u | HAMTShard | /Zigzagumptious +bafybeib5ptplpmtmoyif4lxt5erw5diivp6so4s57lvgfigs53d7zpc4uu | HAMTShard | /Zigzagumptious +bafybeiafmairlfkvgqgticndcs4uomkll4h5xotvnnsw2j7cx3rjwrbh7i | HAMTShard | ↳ /Zigzagumptious +bafybeifjo6xz3onvav6cnpjjuroa7vbzegoxgg5jn6smfvgfcpzwzdaipa | HAMTShard | /Zigzagumptious +bafybeied5si546vcp6klpngu77kftitzffdlxk5ajaytadc3p3ptr7tvam | HAMTShard | /Zigzagumptious +bafybeiar4b5hfriv2pmmbj7npemto6bev7sqkn4jpstwqtaudexpetpg4m | HAMTShard | /Zigzagumptious +bafybeifcekpohkm7qaczfwrrzcdz6j3ykwntawdnp7b7fjfech75vxpwf4 | HAMTShard | ↳ /Zigzagumptious +bafybeihnejxifhvlunhcswhghhrbues366h7o4xqp3ziyeicrdt5473za4 | HAMTShard | /Zigzagumptious +bafybeihlijmc2xyfmomw4fx53mlz3gpt6svgfu3vfokm4wmyd4dagfup24 | HAMTShard | /Zigzagumptious +bafybeihjbh7kptdfaoanrkzhdpx2jwjacg5ncdlcbbwqlum6vwqfga5fii | HAMTShard | /Zigzagumptious +bafybeidaz5aob6tdurpnuj2xeqsjkarqudtono6x2kgbzrzv46ugp3g4ju | HAMTShard | /Zigzagumptious +``` + +#### block + +[testmark]:# (test/hamt_in_directory/block/query) +``` +/ipfs/bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq/Zigzagumptious?dag-scope=block +``` + +[testmark]:# (test/hamt_in_directory/block/execution) +``` +bafybeifrrglx2issn2had5rtstn3xltla6vxmpjfwfz7o3hapvkynh4zoq | Directory | / +bafybeie3mbyp7k77vrpiduaifyhdeqvusjn7qofzwtsp47yavyyb62z32y | HAMTShard | ↳ /Zigzagumptious +``` diff --git a/pkg/retriever/bitswapretriever.go b/pkg/retriever/bitswapretriever.go index b66d9d38..e6b474ca 100644 --- a/pkg/retriever/bitswapretriever.go +++ b/pkg/retriever/bitswapretriever.go @@ -18,8 +18,6 @@ import ( "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/bitswap/client" - "github.com/ipfs/go-libipfs/bitswap/network" "github.com/ipfs/go-unixfsnode" dagpb "github.com/ipld/go-codec-dagpb" "github.com/ipld/go-ipld-prime/datamodel" diff --git a/pkg/server/http/util.go b/pkg/server/http/util.go index 13be0574..f5c7e9b5 100644 --- a/pkg/server/http/util.go +++ b/pkg/server/http/util.go @@ -14,29 +14,11 @@ import ( // parameter is not one of the supported values. func ParseScope(req *http.Request) (types.DagScope, error) { if req.URL.Query().Has("dag-scope") { - switch req.URL.Query().Get("dag-scope") { - case "all": - return types.DagScopeAll, nil - case "entity": - return types.DagScopeEntity, nil - case "block": - return types.DagScopeBlock, nil - default: - return types.DagScopeAll, errors.New("invalid dag-scope parameter") - } + return types.ParseDagScope(req.URL.Query().Get("dag-scope")) } // check for legacy param name -- to do -- delete once we confirm this isn't used any more if req.URL.Query().Has("car-scope") { - switch req.URL.Query().Get("car-scope") { - case "all": - return types.DagScopeAll, nil - case "file": - return types.DagScopeEntity, nil - case "block": - return types.DagScopeBlock, nil - default: - return types.DagScopeAll, errors.New("invalid car-scope parameter") - } + return types.ParseDagScope(req.URL.Query().Get("car-scope")) } return types.DagScopeAll, nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index a6472068..90b52c54 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "math" "net/url" + "strconv" + "strings" "time" "github.com/filecoin-project/go-state-types/abi" @@ -309,6 +312,62 @@ func (ds DagScope) TerminalSelectorSpec() builder.SelectorSpec { panic(fmt.Sprintf("unknown DagScope: [%s]", string(ds))) } +func ParseDagScope(s string) (DagScope, error) { + switch s { + case "all": + return DagScopeAll, nil + case "entity": + return DagScopeEntity, nil + case "block": + return DagScopeBlock, nil + default: + return DagScopeAll, errors.New("invalid dag-scope") + } +} + func (ds DagScope) AcceptHeader() string { return "application/vnd.ipld.car;version=1;order=dfs;dups=y" } + +type ByteRange struct { + From int64 + To int64 +} + +func (br ByteRange) String() string { + to := strconv.FormatInt(br.To, 10) + if br.To == math.MaxInt64 { + to = "*" + } + return fmt.Sprintf("%d:%s", br.From, to) +} + +func (br ByteRange) IsDefault() bool { + return br.From == 0 && br.To == math.MaxInt64 +} + +func ParseByteRange(s string) (ByteRange, error) { + br := ByteRange{From: 0, To: math.MaxInt64} + if s == "" { + return br, nil + } + parts := strings.Split(s, ":") + if len(parts) != 2 { + return br, fmt.Errorf("invalid byte range: %s", s) + } + var err error + br.From, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return br, err + } + if br.From < 0 { + return br, fmt.Errorf("invalid byte range: %s", s) + } + if parts[1] != "*" { + br.To, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return br, err + } + } + return br, nil +} diff --git a/pkg/verifiedcar/verifiedcar_test.go b/pkg/verifiedcar/verifiedcar_test.go index 42c152d4..94b044dd 100644 --- a/pkg/verifiedcar/verifiedcar_test.go +++ b/pkg/verifiedcar/verifiedcar_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" + "github.com/filecoin-project/lassie/pkg/internal/fixtures" "github.com/filecoin-project/lassie/pkg/internal/testutil" + "github.com/filecoin-project/lassie/pkg/types" "github.com/filecoin-project/lassie/pkg/verifiedcar" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -30,6 +32,76 @@ import ( "github.com/stretchr/testify/require" ) +func TestUnixfs20mVariety(t *testing.T) { + req := require.New(t) + + testCases, err := fixtures.Unixfs20mVarietyCases() + req.NoError(err) + storage, closer, err := fixtures.Unixfs20mVarietyReadableStorage() + req.NoError(err) + defer closer.Close() + + lsys := cidlink.DefaultLinkSystem() + lsys.TrustedStorage = true + unixfsnode.AddUnixFSReificationToLinkSystem(&lsys) + lsys.SetReadStorage(storage) + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + req := require.New(t) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + t.Logf("query=%s, blocks=%d", tc.AsQuery(), len(tc.ExpectedCids)) + + // tc.ExpectedCids is in the order we expect to see them in a properly + // formed trustless CAR for the given query. So we build our list of + // expected blocks in that order and feed it through makeCarStream to + // produce the expected CAR. + expectedBlocks := make([]expectedBlock, len(tc.ExpectedCids)) + for ii, ec := range tc.ExpectedCids { + byt, err := lsys.LoadRaw(linking.LinkContext{Ctx: ctx}, cidlink.Link{Cid: ec}) + req.NoError(err) + blk, err := blocks.NewBlockWithCid(byt, ec) + req.NoError(err) + expectedBlocks[ii] = expectedBlock{blk, false} + } + + carStream := makeCarStream(t, ctx, []cid.Cid{tc.Root}, expectedBlocks, false, false, false) + + lsys := cidlink.DefaultLinkSystem() + var writeCounter int + lsys.StorageWriteOpener = func(lc linking.LinkContext) (io.Writer, linking.BlockWriteCommitter, error) { + var buf bytes.Buffer + return &buf, func(l datamodel.Link) error { + req.Equal(expectedBlocks[writeCounter].Cid().String(), l.(cidlink.Link).Cid.String(), "block %d", writeCounter) + req.Equal(expectedBlocks[writeCounter].RawData(), buf.Bytes(), "block %d", writeCounter) + writeCounter++ + return nil + }, nil + } + + // Run the verifier over the CAR stream to see if we end up with + // the same query. + cfg := verifiedcar.Config{ + Root: tc.Root, + Selector: types.PathScopeSelector(tc.Path, tc.Scope), + } + blockCount, byteCount, err := cfg.VerifyCar(ctx, carStream, lsys) + + req.NoError(err) + req.Equal(count(expectedBlocks), blockCount) + req.Equal(sizeOf(expectedBlocks), byteCount) + req.Equal(int(count(expectedBlocks)), writeCounter) + + // Make sure we consumed the entire stream. + byt, err := io.ReadAll(carStream) + req.NoError(err) + req.Equal(0, len(byt)) + }) + } +} + func TestVerifiedCar(t *testing.T) { ctx := context.Background()