From e4d03a6a886e8ef14ebe005d13654263f33a7089 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 5 Apr 2021 12:55:35 -0700 Subject: [PATCH 01/32] switch dag put cmd to directly use prime --- core/commands/dag/put.go | 72 ++++++++++++++++++++++++++++++++++------ go.mod | 2 ++ go.sum | 7 ++++ 3 files changed, 71 insertions(+), 10 deletions(-) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 7f6e744c872..fee0fc9e6ab 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -1,15 +1,21 @@ package dagcmd import ( + "bytes" "fmt" "math" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "github.com/ipfs/go-ipfs/core/commands/cmdenv" - "github.com/ipfs/go-ipfs/core/coredag" + ipldlegacy "github.com/ipfs/go-ipld-legacy" + "github.com/ipld/go-ipld-prime/multicodec" + basicnode "github.com/ipld/go-ipld-prime/node/basic" cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" + mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" ) @@ -36,6 +42,30 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e } } + icodec, ok := mc.Of(ienc) + if !ok { + return fmt.Errorf("%s is not a valid codec name", ienc) + } + fcodec, ok := mc.Of(format) + if !ok { + return fmt.Errorf("%s is not a valid codec name", format) + } + cidPrefix := cid.Prefix{ + Version: 1, + Codec: uint64(fcodec), + MhType: mhType, + MhLength: -1, + } + + decoder, err := multicodec.LookupDecoder(uint64(icodec)) + if err != nil { + return err + } + encoder, err := multicodec.LookupEncoder(uint64(fcodec)) + if err != nil { + return err + } + var adder ipld.NodeAdder = api.Dag() if dopin { adder = api.Dag().Pinning() @@ -48,22 +78,44 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e if file == nil { return fmt.Errorf("expected a regular file") } - nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1) + + node := basicnode.Prototype.Any.NewBuilder() + if err := decoder(node, file); err != nil { + return err + } + n := node.Build() + + bd := bytes.NewBuffer([]byte{}) + if err := encoder(n, bd); err != nil { + return err + } + + blockCid, err := cidPrefix.Sum(bd.Bytes()) + if err != nil { + return err + } + blk, err := blocks.NewBlockWithCid(bd.Bytes(), blockCid) if err != nil { return err } - if len(nds) == 0 { - return fmt.Errorf("no node returned from ParseInputs") + ln := ipldlegacy.LegacyNode{ + Block: blk, + Node: n, } - for _, nd := range nds { - err := b.Add(req.Context, nd) - if err != nil { - return err - } + if err := b.Add(req.Context, &ln); err != nil { + return err } + /* + for _, nd := range nds { + err := b.Add(req.Context, nd) + if err != nil { + return err + } + } + */ - cid := nds[0].Cid() + cid := ln.Cid() if err := res.Emit(&OutputObject{Cid: cid}); err != nil { return err } diff --git a/go.mod b/go.mod index d67cd91587e..4050d430066 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-ipld-git v0.0.3 + github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 // indirect github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-merkledag v0.3.3-0.20210325015807-e952d22343f9 @@ -96,6 +97,7 @@ require ( github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 + github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee // indirect github.com/multiformats/go-multihash v0.0.15 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index f8ca1e9d161..752af82e80a 100644 --- a/go.sum +++ b/go.sum @@ -512,6 +512,7 @@ github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXb github.com/ipld/go-codec-dagpb v1.2.1-0.20210330082435-8ec6b0fbad18/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 h1:m/ZZEoOdswHrrcikTC+fX4x6tnevJs0hoyNzijlT41A= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= +github.com/ipld/go-ipld-prime v0.7.0/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= github.com/ipld/go-ipld-prime v0.7.1-0.20210225173718-8fef5312eb12/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= github.com/ipld/go-ipld-prime v0.7.1-0.20210312004928-8a500e6b8a62/go.mod h1:ZwznT3awHhuBoB0SgGxSo8SZEg6XbQtsZ6WahL9r9z0= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= @@ -961,6 +962,10 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multicodec v0.2.0 h1:MUzKZWxOFagwLLtlx96pub9zwDQAbMAf1k9fXOdc3so= +github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= +github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee h1:vtZuPSrgEZgX575pCi849q2ms6HIsyVSDWVwE7Mqr2g= +github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1423,6 +1428,8 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From f95219e65b35a56c013f21b6eb9e1d077b107aca Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 5 Apr 2021 15:50:45 -0700 Subject: [PATCH 02/32] enable codecs --- core/commands/dag/put.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index fee0fc9e6ab..14b6d8d6886 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -17,6 +17,14 @@ import ( ipld "github.com/ipfs/go-ipld-format" mc "github.com/multiformats/go-multicodec" mh "github.com/multiformats/go-multihash" + + // Expected minimal set of available format/ienc codecs. + _ "github.com/ipld/go-codec-dagpb" + _ "github.com/ipld/go-ipld-prime/codec/cbor" + _ "github.com/ipld/go-ipld-prime/codec/dagcbor" + _ "github.com/ipld/go-ipld-prime/codec/dagjson" + _ "github.com/ipld/go-ipld-prime/codec/json" + _ "github.com/ipld/go-ipld-prime/codec/raw" ) func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { From 92fce1365479a89d7b478999227fb8dfc62629da Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 23 Mar 2021 19:42:28 -0700 Subject: [PATCH 03/32] point api tests to ipld-in-ipfs branch --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8891a342da8..8c7cc07e949 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -229,7 +229,7 @@ jobs: - run: name: Cloning command: | - git clone https://github.com/ipfs/go-ipfs-api.git + git clone -b ipld-in-ipfs https://github.com/ipfs/go-ipfs-api.git git -C go-ipfs-api log -1 - run: name: Starting the daemon From ae20ee7a4786573545fc41d10e4f18a626f7c337 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 5 Apr 2021 15:52:02 -0700 Subject: [PATCH 04/32] partial update to sharness. update git plugin --- core/commands/dag/dag.go | 4 +-- core/commands/dag/put.go | 25 +++++++++++----- go.mod | 6 ++-- go.sum | 8 ++--- plugin/ipld.go | 3 ++ plugin/plugins/git/git.go | 50 ++++++++----------------------- test/sharness/t0053-dag.sh | 6 ++-- test/sharness/t0280-plugin-git.sh | 2 +- 8 files changed, 43 insertions(+), 61 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 005847039d7..71a5ae4d07c 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -77,8 +77,8 @@ into an object of the specified format. cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), }, Options: []cmds.Option{ - cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("cbor"), - cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("json"), + cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("dag-cbor"), + cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), cmds.StringOption("hash", "Hash function to use").WithDefault(""), }, diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 14b6d8d6886..7c6c55c21cc 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "math" + "strconv" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -42,22 +43,21 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e // default hash' (sha256 for cbor, sha1 for git..) mhType := uint64(math.MaxUint64) - if hash != "" { - var ok bool - mhType, ok = mh.Names[hash] - if !ok { - return fmt.Errorf("%s in not a valid multihash name", hash) - } - } - icodec, ok := mc.Of(ienc) if !ok { + if n, err := strconv.Atoi(ienc); err == nil { + icodec = mc.Code(n) + } return fmt.Errorf("%s is not a valid codec name", ienc) } fcodec, ok := mc.Of(format) if !ok { + if n, err := strconv.Atoi(format); err == nil { + fcodec = mc.Code(n) + } return fmt.Errorf("%s is not a valid codec name", format) } + cidPrefix := cid.Prefix{ Version: 1, Codec: uint64(fcodec), @@ -65,6 +65,15 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e MhLength: -1, } + if hash != "" { + var ok bool + mhType, ok = mh.Names[hash] + if !ok { + return fmt.Errorf("%s in not a valid multihash name", hash) + } + cidPrefix.MhType = mhType + } + decoder, err := multicodec.LookupDecoder(uint64(icodec)) if err != nil { return err diff --git a/go.mod b/go.mod index 4050d430066..6dc1ade5046 100644 --- a/go.mod +++ b/go.mod @@ -46,8 +46,8 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-ipld-git v0.0.3 - github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 // indirect + github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461 + github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-merkledag v0.3.3-0.20210325015807-e952d22343f9 @@ -97,7 +97,7 @@ require ( github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 - github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee // indirect + github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee github.com/multiformats/go-multihash v0.0.15 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 752af82e80a..8e3e0a342f0 100644 --- a/go.sum +++ b/go.sum @@ -449,10 +449,10 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-git v0.0.3 h1:/YjkjCyo5KYRpW+suby8Xh9Cm/iH9dAgGV6qyZ1dGus= -github.com/ipfs/go-ipld-git v0.0.3/go.mod h1:RuvMXa9qtJpDbqngyICCU/d+cmLFXxLsbIclmD0Lcr0= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= +github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461 h1:bb97BkhX2Mc9EbwwYLK/rDXoofhkSjpALO//rlxgUw8= +github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= github.com/ipfs/go-ipns v0.0.2/go.mod h1:WChil4e0/m9cIINWLxZe1Jtf77oz5L05rO2ei/uKJ5U= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= @@ -962,8 +962,6 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multicodec v0.2.0 h1:MUzKZWxOFagwLLtlx96pub9zwDQAbMAf1k9fXOdc3so= -github.com/multiformats/go-multicodec v0.2.0/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee h1:vtZuPSrgEZgX575pCi849q2ms6HIsyVSDWVwE7Mqr2g= github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -1428,8 +1426,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= -golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/plugin/ipld.go b/plugin/ipld.go index 5b45e9cd333..38d1484208a 100644 --- a/plugin/ipld.go +++ b/plugin/ipld.go @@ -8,6 +8,9 @@ import ( // PluginIPLD is an interface that can be implemented to add handlers for // for different IPLD formats +// Deprecated: Codecs can now be registered directly in a Plugin's Init method +// using github.com/ipld/go-ipld-prime/multicodec.RegisterEncoder and +// github.com/ipld/go-ipld-prime/multicodec.RegisterDecoder. type PluginIPLD interface { Plugin diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 23b79ad59a1..3859cddaa79 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -2,17 +2,15 @@ package git import ( "compress/zlib" - "fmt" "io" - "math" - "github.com/ipfs/go-ipfs/core/coredag" "github.com/ipfs/go-ipfs/plugin" - "github.com/ipfs/go-cid" - "github.com/ipfs/go-ipld-format" + // Note that depending on this package registers it's multicodec encoder and decoder. git "github.com/ipfs/go-ipld-git" - mh "github.com/multiformats/go-multihash" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + mc "github.com/multiformats/go-multicodec" ) // Plugins is exported list of plugins that will be loaded @@ -22,7 +20,7 @@ var Plugins = []plugin.Plugin{ type gitPlugin struct{} -var _ plugin.PluginIPLD = (*gitPlugin)(nil) +var _ plugin.Plugin = (*gitPlugin)(nil) func (*gitPlugin) Name() string { return "ipld-git" @@ -33,43 +31,19 @@ func (*gitPlugin) Version() string { } func (*gitPlugin) Init(_ *plugin.Environment) error { + // register a custom identifier for import of "zlib-incoded git objects." + // TODO: give this a name. + multicodec.RegisterDecoder(uint64(0x10000000+mc.GitRaw), decodeZlibGit) return nil } -func (*gitPlugin) RegisterBlockDecoders(dec format.BlockDecoder) error { - dec.Register(cid.GitRaw, git.DecodeBlock) - return nil -} - -func (*gitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error { - iec.AddParser("raw", "git", parseRawGit) - iec.AddParser("zlib", "git", parseZlibGit) - return nil -} - -func parseRawGit(r io.Reader, mhType uint64, mhLen int) ([]format.Node, error) { - if mhType != math.MaxUint64 && mhType != mh.SHA1 { - return nil, fmt.Errorf("unsupported mhType %d", mhType) - } - - if mhLen != -1 && mhLen != mh.DefaultLengths[mh.SHA1] { - return nil, fmt.Errorf("invalid mhLen %d", mhLen) - } - - nd, err := git.ParseObject(r) - if err != nil { - return nil, err - } - - return []format.Node{nd}, nil -} - -func parseZlibGit(r io.Reader, mhType uint64, mhLen int) ([]format.Node, error) { +func decodeZlibGit(na ipld.NodeAssembler, r io.Reader) error { rc, err := zlib.NewReader(r) if err != nil { - return nil, err + return err } defer rc.Close() - return parseRawGit(rc, mhType, mhLen) + + return git.Decode(na, rc) } diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 2225f79ec4d..5ae4e2e1deb 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -138,13 +138,13 @@ test_dag_cmd() { ' test_expect_success "non-canonical cbor input is normalized" ' - HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=cbor --input-enc=raw) && + HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=raw) && test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || test_fsh echo $HASH ' test_expect_success "non-canonical cbor input is normalized with input-enc cbor" ' - HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=cbor --input-enc=cbor) && + HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=dag-cbor) && test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || test_fsh echo $HASH ' @@ -174,7 +174,7 @@ test_dag_cmd() { test_expect_success "dag put with json dag-pb works" ' ipfs dag get $HASH > pbjson && - cat pbjson | ipfs dag put --format=dag-pb --input-enc=json > dag_put_out + cat pbjson | ipfs dag put --format=dag-pb --input-enc=dag-json > dag_put_out ' test_expect_success "dag put with dag-pb works output looks good" ' diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index df92e09f590..14d773fce61 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -17,7 +17,7 @@ test_expect_success "prepare test data" ' test_dag_git() { test_expect_success "add objects via dag put" ' - find objects -type f -exec ipfs dag put --format=git --input-enc=zlib {} \; -exec echo \; > hashes + find objects -type f -exec ipfs dag put --format=git --input-enc=268435576 {} \; -exec echo \; > hashes ' test_expect_failure "successfully get added objects" ' From 1a58d05baf68ab2584c4d38d0d78f0cbaab7c1f4 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 24 Mar 2021 18:58:51 -0700 Subject: [PATCH 05/32] fix lint --- core/commands/dag/put.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 7c6c55c21cc..325057affe7 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -47,15 +47,17 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e if !ok { if n, err := strconv.Atoi(ienc); err == nil { icodec = mc.Code(n) + } else { + return fmt.Errorf("%s is not a valid codec name", ienc) } - return fmt.Errorf("%s is not a valid codec name", ienc) } fcodec, ok := mc.Of(format) if !ok { if n, err := strconv.Atoi(format); err == nil { fcodec = mc.Code(n) + } else { + return fmt.Errorf("%s is not a valid codec name", format) } - return fmt.Errorf("%s is not a valid codec name", format) } cidPrefix := cid.Prefix{ From 474fdcfab50b589a34c2b7b218bb8904bf86794c Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 25 Mar 2021 15:08:08 -0700 Subject: [PATCH 06/32] correct codec name --- test/sharness/t0280-plugin-git.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index 14d773fce61..4fee0b6aecb 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -17,7 +17,7 @@ test_expect_success "prepare test data" ' test_dag_git() { test_expect_success "add objects via dag put" ' - find objects -type f -exec ipfs dag put --format=git --input-enc=268435576 {} \; -exec echo \; > hashes + find objects -type f -exec ipfs dag put --format=git-raw --input-enc=268435576 {} \; -exec echo \; > hashes ' test_expect_failure "successfully get added objects" ' From d84bb28f69f43cb52089939e77bc35d9d79511af Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 25 Mar 2021 15:35:28 -0700 Subject: [PATCH 07/32] add reasonable default mhType. --- core/commands/dag/put.go | 6 ++---- test/sharness/t0053-dag.sh | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 325057affe7..aea907e3e41 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -3,7 +3,6 @@ package dagcmd import ( "bytes" "fmt" - "math" "strconv" blocks "github.com/ipfs/go-block-format" @@ -39,9 +38,8 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) - // mhType tells inputParser which hash should be used. MaxUint64 means 'use - // default hash' (sha256 for cbor, sha1 for git..) - mhType := uint64(math.MaxUint64) + // mhType tells inputParser which hash should be used. Default otherwise is sha256 + mhType := uint64(mh.SHA2_256) icodec, ok := mc.Of(ienc) if !ok { diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 5ae4e2e1deb..8f7d8e2674d 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -27,7 +27,7 @@ test_expect_success "make an ipld object in json" ' test_dag_cmd() { test_expect_success "can add an ipld object using protobuf" ' - IPLDHASH=$(cat ipld_object | ipfs dag put -f protobuf) + IPLDHASH=$(cat ipld_object | ipfs dag put -i dag-json -f dag-pb) ' test_expect_success "output looks correct" ' @@ -36,7 +36,7 @@ test_dag_cmd() { ' test_expect_success "can add an ipld object using protobuf and --cid=base=base32" ' - IPLDHASHb32=$(cat ipld_object | ipfs dag put -f protobuf --cid-base=base32) + IPLDHASHb32=$(cat ipld_object | ipfs dag put -i dag-json -f dag-pb --cid-base=base32) ' test_expect_success "output looks correct (does not upgrade to CIDv1)" ' From 71ea0d6adf01b2efe1f4509fcefb8b57bbecc9bf Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 25 Mar 2021 17:01:08 -0700 Subject: [PATCH 08/32] fix input encoding --- test/sharness/t0053-dag.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 8f7d8e2674d..aa92c7a26b9 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -27,7 +27,7 @@ test_expect_success "make an ipld object in json" ' test_dag_cmd() { test_expect_success "can add an ipld object using protobuf" ' - IPLDHASH=$(cat ipld_object | ipfs dag put -i dag-json -f dag-pb) + IPLDHASH=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-pb) ' test_expect_success "output looks correct" ' @@ -36,7 +36,7 @@ test_dag_cmd() { ' test_expect_success "can add an ipld object using protobuf and --cid=base=base32" ' - IPLDHASHb32=$(cat ipld_object | ipfs dag put -i dag-json -f dag-pb --cid-base=base32) + IPLDHASHb32=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-pb --cid-base=base32) ' test_expect_success "output looks correct (does not upgrade to CIDv1)" ' From 37a128bd2000c2dbebd150992069fb94268c0b0e Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 5 Apr 2021 15:52:34 -0700 Subject: [PATCH 09/32] update go-ipld-git to encode from basic nodes --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6dc1ade5046..a4156792f4d 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461 + github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 diff --git a/go.sum b/go.sum index 8e3e0a342f0..9c8c29ae1e2 100644 --- a/go.sum +++ b/go.sum @@ -453,6 +453,10 @@ github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2 github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461 h1:bb97BkhX2Mc9EbwwYLK/rDXoofhkSjpALO//rlxgUw8= github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= +github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc h1:57luC/Oc36sZBp4BAJ74MJGsAULBeQ5NLLRpTL6YJwI= +github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= +github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 h1:5RLoebNIGmHmlcdTSdVrXGRF5sLy/wTiCUgWMeJvIII= +github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5/go.mod h1:1wv+3vTMw6axVoAJFisWzJ7tewUnrcr6AxymcZrd83k= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= github.com/ipfs/go-ipns v0.0.2/go.mod h1:WChil4e0/m9cIINWLxZe1Jtf77oz5L05rO2ei/uKJ5U= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= From ac1490dfac23f1597ed5337ab8af8700947b7d20 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 26 Mar 2021 11:10:10 -0700 Subject: [PATCH 10/32] fix some sharness test --- test/sharness/t0053-dag.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index aa92c7a26b9..937d4985746 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -26,17 +26,17 @@ test_expect_success "make an ipld object in json" ' ' test_dag_cmd() { - test_expect_success "can add an ipld object using protobuf" ' - IPLDHASH=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-pb) + test_expect_success "can add an ipld object using " ' + IPLDHASH=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-cbor) ' test_expect_success "output looks correct" ' - EXPHASH="QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n" + EXPHASH="bafyreicktrf5jt6zu2npbojxx2onikqqdihoj2xutehmkmzrsw5bo5zgge" test $EXPHASH = $IPLDHASH ' test_expect_success "can add an ipld object using protobuf and --cid=base=base32" ' - IPLDHASHb32=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-pb --cid-base=base32) + IPLDHASHb32=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-cbor --cid-base=base32) ' test_expect_success "output looks correct (does not upgrade to CIDv1)" ' @@ -48,7 +48,7 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - EXPHASH="bafyreidjtjfmavdk7epvztob2m5vlm3pxp3gjmpyewro4qlbw5n4f4iz64" + EXPHASH="bafyreicktrf5jt6zu2npbojxx2onikqqdihoj2xutehmkmzrsw5bo5zgge" test $EXPHASH = $IPLDHASH ' From 040c8f3796b248d3f10e4a1e96987487097a2017 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 5 Apr 2021 12:51:32 -0700 Subject: [PATCH 11/32] nits --- core/commands/dag/put.go | 12 ++++++------ test/sharness/t0051-object.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index aea907e3e41..2562d69d880 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -43,19 +43,19 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e icodec, ok := mc.Of(ienc) if !ok { - if n, err := strconv.Atoi(ienc); err == nil { - icodec = mc.Code(n) - } else { + n, err := strconv.Atoi(ienc) + if err != nil { return fmt.Errorf("%s is not a valid codec name", ienc) } + icodec = mc.Code(n) } fcodec, ok := mc.Of(format) if !ok { - if n, err := strconv.Atoi(format); err == nil { - fcodec = mc.Code(n) - } else { + n, err := strconv.Atoi(format) + if err != nil { return fmt.Errorf("%s is not a valid codec name", format) } + fcodec = mc.Code(n) } cidPrefix := cid.Prefix{ diff --git a/test/sharness/t0051-object.sh b/test/sharness/t0051-object.sh index 95de4fda758..82e746ddec8 100755 --- a/test/sharness/t0051-object.sh +++ b/test/sharness/t0051-object.sh @@ -130,7 +130,7 @@ test_object_cmd() { ' test_expect_failure "'ipfs object put --inputenc=xml' output looks good" ' - echo "added $HASH\n" >expected && + echo "added $HASH" >expected && test_cmp expected actual ' From 0d0f43c0cfe32c9a949341aff449219140ca6811 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 6 Apr 2021 20:20:29 -0700 Subject: [PATCH 12/32] pb sharness update --- test/sharness/t0053-dag.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 937d4985746..6d099e3e7a5 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -25,11 +25,20 @@ test_expect_success "make an ipld object in json" ' printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"},\"sub\":{\"dict\":\"ionary\",\"beep\":[0,\"bop\"]}}" $HASH1 $HASH2 $HASH3 > ipld_object ' +test_expect_success "make an ipld object in dag-pb" ' + printf "{\"data\":\"thisisatest\",\"links\":[{\"/\":\"%s\"},{\"/\":\"%s\"}]}" $HASH1 $HASH2 > ipld_object_pb +' + + test_dag_cmd() { - test_expect_success "can add an ipld object using " ' + test_expect_success "can add an ipld object using dag-json" ' IPLDHASH=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-cbor) ' + test_expect_success "can add an ipld object using dag-json" ' + IPLDPBHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-json -f dag-pb) + ' + test_expect_success "output looks correct" ' EXPHASH="bafyreicktrf5jt6zu2npbojxx2onikqqdihoj2xutehmkmzrsw5bo5zgge" test $EXPHASH = $IPLDHASH @@ -103,7 +112,7 @@ test_dag_cmd() { ' test_expect_success "can get object" ' - ipfs dag get $IPLDHASH > ipld_obj_out + ipfs dag get $IPLDPBHASH > ipld_obj_out ' test_expect_success "object links look right" ' @@ -112,7 +121,7 @@ test_dag_cmd() { test_expect_success "retrieved object hashes back correctly" ' IPLDHASH2=$(cat ipld_obj_out | ipfs dag put) && - test "$IPLDHASH" = "$IPLDHASH2" + test "$IPLDPBHASH" = "$IPLDHASH2" ' test_expect_success "add a normal file" ' From 4b3013af4f7c155fabb3f2b17a7e22ee97ca62c5 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Wed, 7 Apr 2021 08:55:46 -0700 Subject: [PATCH 13/32] try with capitalized fields --- test/sharness/t0053-dag.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 6d099e3e7a5..95dc0476519 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -26,7 +26,7 @@ test_expect_success "make an ipld object in json" ' ' test_expect_success "make an ipld object in dag-pb" ' - printf "{\"data\":\"thisisatest\",\"links\":[{\"/\":\"%s\"},{\"/\":\"%s\"}]}" $HASH1 $HASH2 > ipld_object_pb + printf "{\"Data\":\"thisisatest\",\"Links\":[{\"/\":\"%s\"},{\"/\":\"%s\"}]}" $HASH1 $HASH2 > ipld_object_pb ' From c2372fcfa13665002b43f06fecb7ff32a8d443d4 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Fri, 9 Apr 2021 14:01:07 -0700 Subject: [PATCH 14/32] dag-json bytes --- test/sharness/t0053-dag.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 95dc0476519..e4bd6af25b6 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -26,7 +26,8 @@ test_expect_success "make an ipld object in json" ' ' test_expect_success "make an ipld object in dag-pb" ' - printf "{\"Data\":\"thisisatest\",\"Links\":[{\"/\":\"%s\"},{\"/\":\"%s\"}]}" $HASH1 $HASH2 > ipld_object_pb + B64=$(cat ipld_object | base64) + printf "{\"Data\":{\"/\":{\"bytes\":\"%s\"}},\"Links\":[{\"/\":\"%s\"},{\"/\":\"%s\"}]}" $B64 $HASH1 $HASH2 > ipld_object_pb ' From 9fe56d8405ebf31ab867a01419ffc4fcf2968e75 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 12 Apr 2021 10:53:47 -0700 Subject: [PATCH 15/32] use dag-cbor that can represent bytes --- test/sharness/t0053-dag.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index e4bd6af25b6..08945c8741a 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -25,9 +25,8 @@ test_expect_success "make an ipld object in json" ' printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"},\"sub\":{\"dict\":\"ionary\",\"beep\":[0,\"bop\"]}}" $HASH1 $HASH2 $HASH3 > ipld_object ' -test_expect_success "make an ipld object in dag-pb" ' - B64=$(cat ipld_object | base64) - printf "{\"Data\":{\"/\":{\"bytes\":\"%s\"}},\"Links\":[{\"/\":\"%s\"},{\"/\":\"%s\"}]}" $B64 $HASH1 $HASH2 > ipld_object_pb +test_expect_success "make an ipld object in dag-json" ' + echo "omREYXRhT6JkRGF0YWF4ZUxpbmtzgGVMaW5rc4A=" | base64 -d > ipld_object_pb ' @@ -37,7 +36,7 @@ test_dag_cmd() { ' test_expect_success "can add an ipld object using dag-json" ' - IPLDPBHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-json -f dag-pb) + IPLDPBHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-cbor -f dag-pb) ' test_expect_success "output looks correct" ' From 13fb376f3c8fb5651ba94909683f345f97ad547d Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 13 Apr 2021 18:12:24 -0700 Subject: [PATCH 16/32] additional work on sharness --- test/sharness/t0053-dag.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 08945c8741a..7d64d5d7e9a 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -39,6 +39,10 @@ test_dag_cmd() { IPLDPBHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-cbor -f dag-pb) ' + test_expect_success "can add an ipld object using dag-json" ' + IPLDCBORHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-cbor -f dag-cbor) + ' + test_expect_success "output looks correct" ' EXPHASH="bafyreicktrf5jt6zu2npbojxx2onikqqdihoj2xutehmkmzrsw5bo5zgge" test $EXPHASH = $IPLDHASH @@ -106,6 +110,10 @@ test_dag_cmd() { ipfs pin add $EXPHASH ' + test_expect_success "can pin dag object" ' + ipfs pin add $IPLDPBHASH + ' + test_expect_success "after gc, objects still accessible" ' ipfs repo gc > /dev/null && ipfs refs -r --timeout=2s $EXPHASH > /dev/null @@ -120,8 +128,8 @@ test_dag_cmd() { ' test_expect_success "retrieved object hashes back correctly" ' - IPLDHASH2=$(cat ipld_obj_out | ipfs dag put) && - test "$IPLDPBHASH" = "$IPLDHASH2" + IPLDHASH2=$(cat ipld_obj_out | ipfs dag put --input-enc=dag-json -f dag-cbor) && + test "$IPLDCBORHASH" = "$IPLDHASH2" ' test_expect_success "add a normal file" ' @@ -147,7 +155,7 @@ test_dag_cmd() { ' test_expect_success "non-canonical cbor input is normalized" ' - HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=raw) && + HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=dag-cbor) && test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || test_fsh echo $HASH ' @@ -159,7 +167,7 @@ test_dag_cmd() { ' test_expect_success "add an ipld with pin" ' - PINHASH=$(printf {\"foo\":\"bar\"} | ipfs dag put --pin=true) + PINHASH=$(printf {\"foo\":\"bar\"} | ipfs dag put --input-enc=dag-json --pin=true) ' test_expect_success "after gc, objects still accessible" ' From e1ddb1a8de43dd61dcfccec0e896bb3e3edb3f63 Mon Sep 17 00:00:00 2001 From: Will Scott Date: Thu, 15 Apr 2021 22:35:49 -0700 Subject: [PATCH 17/32] fix dag-cbor fixture sharness test --- test/sharness/t0053-dag.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 7d64d5d7e9a..755fb406ab1 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -156,14 +156,13 @@ test_dag_cmd() { test_expect_success "non-canonical cbor input is normalized" ' HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=dag-cbor) && - test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || + test $HASH = "bafyreicfngo3hexoxbbwsm64apbydw5fyeruyrfwiyxlj3f4wuzdyztd4m" || test_fsh echo $HASH ' - test_expect_success "non-canonical cbor input is normalized with input-enc cbor" ' - HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=dag-cbor) && - test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || - test_fsh echo $HASH + test_expect_success "cbor input can be fetched" ' + EXPARR=$(ipfs dag get $HASH/arr) + test $EXPARR = "[]" ' test_expect_success "add an ipld with pin" ' @@ -180,7 +179,7 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - EXPHASH="bafyriqgae54zjl3bjebmbat2rjem4ewj6vni6jxohandmvk3bibfgv3sioyeidppsghvulryxats43br3b7afa6jy77x6gqzqaicer6ljicck" + EXPHASH="bafyriqf6pe376nnoh4ngsxmo7inplk35k5ucdfwvbc547vhp5mxx6wknllub3nyk44pvp37kgzrp2cotebl2onrqg5i7an4rcuu66cnj4ij4i" test $EXPHASH = $IPLDHASH ' From f57a8c5ea017d928b4d992f344d2071d6d5dd50d Mon Sep 17 00:00:00 2001 From: Will Scott Date: Mon, 12 Jul 2021 16:39:55 -0700 Subject: [PATCH 18/32] Switch `ipfs dag get` to serialize to an IPLD-Prime codec and output in the same way as `ipfs cat` --- core/commands/dag/dag.go | 3 +++ core/commands/dag/get.go | 50 +++++++++++++++++++++++++++++++++++----- go.mod | 2 +- go.sum | 6 +++-- 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 71a5ae4d07c..2e7aebb7daf 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -108,6 +108,9 @@ format. Arguments: []cmds.Argument{ cmds.StringArg("ref", true, false, "The object to get").EnableStdin(), }, + Options: []cmds.Option{ + cmds.StringOption("format", "f", "Format that the object will be serialized as.").WithDefault("dag-json"), + }, Run: dagGet, } diff --git a/core/commands/dag/get.go b/core/commands/dag/get.go index a5f92273ae3..cef5605e187 100644 --- a/core/commands/dag/get.go +++ b/core/commands/dag/get.go @@ -1,11 +1,19 @@ package dagcmd import ( - "strings" + "fmt" + "io" + "strconv" "github.com/ipfs/go-ipfs/core/commands/cmdenv" + ipldlegacy "github.com/ipfs/go-ipld-legacy" "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + "github.com/ipld/go-ipld-prime/traversal" + mc "github.com/multiformats/go-multicodec" + cmds "github.com/ipfs/go-ipfs-cmds" ) @@ -15,6 +23,16 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } + format, _ := req.Options["format"].(string) + fCodec, ok := mc.Of(format) + if !ok { + n, err := strconv.Atoi(format) + if err != nil { + return fmt.Errorf("%s is not a valid codec name", format) + } + fCodec = mc.Code(n) + } + rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0])) if err != nil { return err @@ -25,14 +43,34 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e return err } - var out interface{} = obj + universal, ok := obj.(ipldlegacy.UniversalNode) + if !ok { + return fmt.Errorf("%T is not a valid IPLD node", obj) + } + + finalNode := universal.(ipld.Node) + if len(rp.Remainder()) > 0 { - rem := strings.Split(rp.Remainder(), "/") - final, _, err := obj.Resolve(rem) + remainderPath := ipld.ParsePath(rp.Remainder()) + + finalNode, err = traversal.Get(finalNode, remainderPath) if err != nil { return err } - out = final } - return cmds.EmitOnce(res, &out) + + encoder, err := multicodec.LookupEncoder(uint64(fCodec)) + if err != nil { + return fmt.Errorf("invalid encoding: %s - %s", format, err) + } + + r, w := io.Pipe() + go func() { + defer w.Close() + if err := encoder(finalNode, w); err != nil { + _ = res.CloseWithError(err) + } + }() + + return res.Emit(r) } diff --git a/go.mod b/go.mod index a4156792f4d..4982558c8fd 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc - github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 + github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 github.com/ipfs/go-merkledag v0.3.3-0.20210325015807-e952d22343f9 diff --git a/go.sum b/go.sum index 9c8c29ae1e2..66aa720a244 100644 --- a/go.sum +++ b/go.sum @@ -449,14 +449,14 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= -github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461 h1:bb97BkhX2Mc9EbwwYLK/rDXoofhkSjpALO//rlxgUw8= github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc h1:57luC/Oc36sZBp4BAJ74MJGsAULBeQ5NLLRpTL6YJwI= github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 h1:5RLoebNIGmHmlcdTSdVrXGRF5sLy/wTiCUgWMeJvIII= github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5/go.mod h1:1wv+3vTMw6axVoAJFisWzJ7tewUnrcr6AxymcZrd83k= +github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= +github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= github.com/ipfs/go-ipns v0.0.2/go.mod h1:WChil4e0/m9cIINWLxZe1Jtf77oz5L05rO2ei/uKJ5U= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= @@ -511,6 +511,8 @@ github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 h1:P github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62/go.mod h1:leFsW1rX9F7+0N9Tss2kR82w9oafDmczBdE8z4VWGRo= github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f h1:417+6v+keLsNoVi2vPJamDsW/ewuAs2uE3hlFt5muGs= github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f/go.mod h1:kHunAcD305JwLqwI9MfKvuQu4ljwWRZQWfDWPrkkhcg= +github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs= +github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs= github.com/ipld/go-codec-dagpb v1.0.2-0.20210308154810-d05d02fa186e/go.mod h1:oYexiw3WkBIVD5UTNkVuOd0iyEcLxqytAQa90F3nH9M= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.2.1-0.20210330082435-8ec6b0fbad18/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= From 319e9c330a00ebb47d6c9321e162a343f240bf3f Mon Sep 17 00:00:00 2001 From: Will Scott Date: Tue, 13 Jul 2021 15:42:48 -0700 Subject: [PATCH 19/32] bump ipld-prime --- go.mod | 2 +- go.sum | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 4982558c8fd..9fa9531b8c2 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/ipfs/go-unixfsnode v1.1.1 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 - github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f + github.com/ipld/go-car v0.3.1 github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1 github.com/jbenet/go-is-domain v1.0.5 diff --git a/go.sum b/go.sum index 66aa720a244..d86d014a0ba 100644 --- a/go.sum +++ b/go.sum @@ -449,12 +449,8 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461 h1:bb97BkhX2Mc9EbwwYLK/rDXoofhkSjpALO//rlxgUw8= -github.com/ipfs/go-ipld-git v0.0.4-0.20210324174925-515bc5cab461/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc h1:57luC/Oc36sZBp4BAJ74MJGsAULBeQ5NLLRpTL6YJwI= github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= -github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5 h1:5RLoebNIGmHmlcdTSdVrXGRF5sLy/wTiCUgWMeJvIII= -github.com/ipfs/go-ipld-legacy v0.0.0-20210312014519-2895f54096d5/go.mod h1:1wv+3vTMw6axVoAJFisWzJ7tewUnrcr6AxymcZrd83k= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= @@ -509,18 +505,12 @@ github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZ github.com/ipfs/interface-go-ipfs-core v0.4.0/go.mod h1:UJBcU6iNennuI05amq3FQ7g0JHUkibHFAfhfUIy927o= github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 h1:PJ5gH7jjJtQ8OT+iACcolpMnvhCpfRtpegWfMF785uk= github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62/go.mod h1:leFsW1rX9F7+0N9Tss2kR82w9oafDmczBdE8z4VWGRo= -github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f h1:417+6v+keLsNoVi2vPJamDsW/ewuAs2uE3hlFt5muGs= -github.com/ipld/go-car v0.2.1-0.20210312021557-7afab98d034f/go.mod h1:kHunAcD305JwLqwI9MfKvuQu4ljwWRZQWfDWPrkkhcg= github.com/ipld/go-car v0.3.1 h1:WT+3cdmXlvmWOlGxk9webhj4auGO5QvgqC2vCCkFRXs= github.com/ipld/go-car v0.3.1/go.mod h1:dPkEWeAK8KaVvH5TahaCs6Mncpd4lDMpkbs0/SPzuVs= -github.com/ipld/go-codec-dagpb v1.0.2-0.20210308154810-d05d02fa186e/go.mod h1:oYexiw3WkBIVD5UTNkVuOd0iyEcLxqytAQa90F3nH9M= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.2.1-0.20210330082435-8ec6b0fbad18/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 h1:m/ZZEoOdswHrrcikTC+fX4x6tnevJs0hoyNzijlT41A= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= -github.com/ipld/go-ipld-prime v0.7.0/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= -github.com/ipld/go-ipld-prime v0.7.1-0.20210225173718-8fef5312eb12/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM= -github.com/ipld/go-ipld-prime v0.7.1-0.20210312004928-8a500e6b8a62/go.mod h1:ZwznT3awHhuBoB0SgGxSo8SZEg6XbQtsZ6WahL9r9z0= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= @@ -1278,12 +1268,10 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1418,7 +1406,6 @@ golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1430,7 +1417,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201223074533-0d417f636930/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY= From 215ec2c7bac3f09d7ff51b30e281ed5f313041c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 19 Jul 2021 00:05:20 +0100 Subject: [PATCH 20/32] bump go-multicodec and use Code.Set --- core/commands/dag/get.go | 11 +++-------- core/commands/dag/put.go | 21 ++++++--------------- go.mod | 4 ++-- go.sum | 8 ++++---- 4 files changed, 15 insertions(+), 29 deletions(-) diff --git a/core/commands/dag/get.go b/core/commands/dag/get.go index cef5605e187..c5c983e09b5 100644 --- a/core/commands/dag/get.go +++ b/core/commands/dag/get.go @@ -3,7 +3,6 @@ package dagcmd import ( "fmt" "io" - "strconv" "github.com/ipfs/go-ipfs/core/commands/cmdenv" ipldlegacy "github.com/ipfs/go-ipld-legacy" @@ -24,13 +23,9 @@ func dagGet(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e } format, _ := req.Options["format"].(string) - fCodec, ok := mc.Of(format) - if !ok { - n, err := strconv.Atoi(format) - if err != nil { - return fmt.Errorf("%s is not a valid codec name", format) - } - fCodec = mc.Code(n) + var fCodec mc.Code + if err := fCodec.Set(format); err != nil { + return err } rp, err := api.ResolvePath(req.Context, path.New(req.Arguments[0])) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 2562d69d880..0bbfbd9df63 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -3,7 +3,6 @@ package dagcmd import ( "bytes" "fmt" - "strconv" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -41,21 +40,13 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e // mhType tells inputParser which hash should be used. Default otherwise is sha256 mhType := uint64(mh.SHA2_256) - icodec, ok := mc.Of(ienc) - if !ok { - n, err := strconv.Atoi(ienc) - if err != nil { - return fmt.Errorf("%s is not a valid codec name", ienc) - } - icodec = mc.Code(n) + var icodec mc.Code + if err := icodec.Set(ienc); err != nil { + return err } - fcodec, ok := mc.Of(format) - if !ok { - n, err := strconv.Atoi(format) - if err != nil { - return fmt.Errorf("%s is not a valid codec name", format) - } - fcodec = mc.Code(n) + var fcodec mc.Code + if err := fcodec.Set(format); err != nil { + return err } cidPrefix := cid.Prefix{ diff --git a/go.mod b/go.mod index 9fa9531b8c2..0a44e6fa31d 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 github.com/ipld/go-car v0.3.1 github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 - github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1 + github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5 github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 @@ -97,7 +97,7 @@ require ( github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 - github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee + github.com/multiformats/go-multicodec v0.2.1-0.20210718225951-a237c4436b76 github.com/multiformats/go-multihash v0.0.15 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index d86d014a0ba..257fec11e4e 100644 --- a/go.sum +++ b/go.sum @@ -512,11 +512,11 @@ github.com/ipld/go-codec-dagpb v1.2.1-0.20210330082435-8ec6b0fbad18/go.mod h1:GM github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 h1:m/ZZEoOdswHrrcikTC+fX4x6tnevJs0hoyNzijlT41A= github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1 h1:dIKSj9r+CCXz9t6p4TfbDx34CfSjLfasRZf37SXlNjg= github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5 h1:hlKWs60jv1TN1WqYS4RueP9+MS2RRCFqj7CgJ+uDFQc= +github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -958,8 +958,8 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee h1:vtZuPSrgEZgX575pCi849q2ms6HIsyVSDWVwE7Mqr2g= -github.com/multiformats/go-multicodec v0.2.1-0.20210319221629-f3663dccaeee/go.mod h1:/y4YVwkfMyry5kFbMTbLJKErhycTIftytRV+llXdyS4= +github.com/multiformats/go-multicodec v0.2.1-0.20210718225951-a237c4436b76 h1:VD/M8CCDR5uOoNpeCSejcHp3MY7XaQYqLr3ANrDDRxQ= +github.com/multiformats/go-multicodec v0.2.1-0.20210718225951-a237c4436b76/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= From 3e8f569d49fe0ad19a2c5c6b5cf4c281b5276cb0 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 15 Jul 2021 21:25:31 +1000 Subject: [PATCH 21/32] fix dag test fixtures for go-ipld-prime codecs --- go.mod | 1 + test/sharness/t0053-dag.sh | 108 +++++++++++++++++++++++-------------- 2 files changed, 69 insertions(+), 40 deletions(-) diff --git a/go.mod b/go.mod index 0a44e6fa31d..2b510e746cb 100644 --- a/go.mod +++ b/go.mod @@ -119,3 +119,4 @@ require ( ) go 1.14 +replace github.com/ipld/go-ipld-prime => ./../../ipld/go-ipld-prime/ \ No newline at end of file diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 755fb406ab1..826a2eda132 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -22,29 +22,31 @@ test_expect_success "make a few test files" ' ' test_expect_success "make an ipld object in json" ' - printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"},\"sub\":{\"dict\":\"ionary\",\"beep\":[0,\"bop\"]}}" $HASH1 $HASH2 $HASH3 > ipld_object + printf "{\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"hello\":\"world\",\"magic\":{\"/\":\"%s\"},\"subNode\":{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}}" $HASH1 $HASH2 $HASH3 > ipld_object ' -test_expect_success "make an ipld object in dag-json" ' - echo "omREYXRhT6JkRGF0YWF4ZUxpbmtzgGVMaW5rc4A=" | base64 -d > ipld_object_pb +test_expect_success "make the same ipld object in dag-cbor, dag-json and dag-pb" ' + echo "omREYXRhT6JkRGF0YWF4ZUxpbmtzgGVMaW5rc4A=" | base64 -d > ipld_object_dagcbor + echo "Cg+iZERhdGFheGVMaW5rc4A=" | base64 -d > ipld_object_dagpb + echo "{\"Data\":{\"/\":{\"bytes\":\"omREYXRhYXhlTGlua3OA\"}},\"Links\":[]}" > ipld_object_dagjson ' - test_dag_cmd() { - test_expect_success "can add an ipld object using dag-json" ' + test_expect_success "can add an ipld object using dag-json to dag-cbor" ' IPLDHASH=$(cat ipld_object | ipfs dag put --input-enc=dag-json -f dag-cbor) ' - test_expect_success "can add an ipld object using dag-json" ' - IPLDPBHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-cbor -f dag-pb) + test_expect_success "CID looks correct" ' + EXPHASH="bafyreihim5n3etxp5om74fdpriqtys5sdmgbd5ix2oil7u2g2qfdpdlgie" + test $EXPHASH = $IPLDHASH ' - test_expect_success "can add an ipld object using dag-json" ' - IPLDCBORHASH=$(cat ipld_object_pb | ipfs dag put --input-enc=dag-cbor -f dag-cbor) + test_expect_success "can add an ipld object using defaults" ' + IPLDHASH=$(cat ipld_object | ipfs dag put) ' - test_expect_success "output looks correct" ' - EXPHASH="bafyreicktrf5jt6zu2npbojxx2onikqqdihoj2xutehmkmzrsw5bo5zgge" + test_expect_success "CID looks correct" ' + EXPHASH="bafyreihim5n3etxp5om74fdpriqtys5sdmgbd5ix2oil7u2g2qfdpdlgie" test $EXPHASH = $IPLDHASH ' @@ -56,21 +58,39 @@ test_dag_cmd() { test $EXPHASH = $IPLDHASHb32 ' - test_expect_success "can add an ipld object" ' - IPLDHASH=$(cat ipld_object | ipfs dag put) + test_expect_success "can add an ipld object using --cid-base=base32" ' + IPLDHASHb32=$(cat ipld_object | ipfs dag put --cid-base=base32) ' test_expect_success "output looks correct" ' - EXPHASH="bafyreicktrf5jt6zu2npbojxx2onikqqdihoj2xutehmkmzrsw5bo5zgge" - test $EXPHASH = $IPLDHASH + test $(ipfs cid base32 $EXPHASH) = $IPLDHASHb32 ' - test_expect_success "can add an ipld object using --cid-base=base32" ' - IPLDHASHb32=$(cat ipld_object | ipfs dag put --cid-base=base32) + test_expect_success "can add an ipld object using dag-cbor to dag-pb" ' + IPLDPBHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-enc=dag-cbor -f dag-pb) ' - test_expect_success "output looks correct" ' - test $(ipfs cid base32 $EXPHASH) = $IPLDHASHb32 + test_expect_success "dag-pb CID looks correct" ' + EXPHASH="bafybeicbvnvha5b25w77plipz3v5axqydbhxb2s2ojglexotowwieoecvy" + test $EXPHASH = $IPLDPBHASH + ' + + test_expect_success "can add an ipld object using dag-cbor to dag-cbor" ' + IPLDCBORHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-enc=dag-cbor -f dag-cbor) + ' + + test_expect_success "dag-cbor CID looks correct" ' + EXPHASH="bafyreibuwyi6b7oruz64krsakf5a4zmttu6dqksu6bpkpy3sjrm6a73o6a" + test $EXPHASH = $IPLDCBORHASH + ' + + test_expect_success "can add an ipld object using dag-cbor to dag-json" ' + IPLDJSONHASH=$(cat ipld_object_dagcbor | ipfs dag put --input-enc=dag-cbor -f dag-json) + ' + + test_expect_success "dag-cbor CID looks correct" ' + EXPHASH="baguqeerarvo2dwtckziergct46ijehc4yc7o3y7rwsiomwf65frbhqtgc4ta" + test $EXPHASH = $IPLDJSONHASH ' test_expect_success "various path traversals work" ' @@ -87,40 +107,48 @@ test_dag_cmd() { test_expect_success "resolving sub-objects works" ' ipfs dag get $IPLDHASH/hello > sub1 && - ipfs dag get $IPLDHASH/sub > sub2 && - ipfs dag get $IPLDHASH/sub/beep > sub3 && - ipfs dag get $IPLDHASH/sub/beep/0 > sub4 && - ipfs dag get $IPLDHASH/sub/beep/1 > sub5 + ipfs dag get $IPLDHASH/subNode > sub2 && + ipfs dag get $IPLDHASH/subNode/beep > sub3 && + ipfs dag get $IPLDHASH/subNode/beep/0 > sub4 && + ipfs dag get $IPLDHASH/subNode/beep/1 > sub5 ' test_expect_success "sub-objects look right" ' - echo "\"world\"" > sub1_exp && + echo -n "\"world\"" > sub1_exp && test_cmp sub1_exp sub1 && - echo "{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}" > sub2_exp && + echo -n "{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}" > sub2_exp && test_cmp sub2_exp sub2 && - echo "[0,\"bop\"]" > sub3_exp && + echo -n "[0,\"bop\"]" > sub3_exp && test_cmp sub3_exp sub3 && - echo "0" > sub4_exp && + echo -n "0" > sub4_exp && test_cmp sub4_exp sub4 && - echo "\"bop\"" > sub5_exp && + echo -n "\"bop\"" > sub5_exp && test_cmp sub5_exp sub5 ' - test_expect_success "can pin cbor object" ' - ipfs pin add $EXPHASH + test_expect_success "can pin ipld object" ' + ipfs pin add $IPLDHASH ' - test_expect_success "can pin dag object" ' + test_expect_success "can pin dag-pb object" ' ipfs pin add $IPLDPBHASH ' + test_expect_success "can pin dag-cbor object" ' + ipfs pin add $IPLDCBORHASH + ' + + test_expect_success "can pin dag-json object" ' + ipfs pin add $IPLDJSONHASH + ' + test_expect_success "after gc, objects still accessible" ' ipfs repo gc > /dev/null && ipfs refs -r --timeout=2s $EXPHASH > /dev/null ' test_expect_success "can get object" ' - ipfs dag get $IPLDPBHASH > ipld_obj_out + ipfs dag get $IPLDHASH > ipld_obj_out ' test_expect_success "object links look right" ' @@ -129,7 +157,7 @@ test_dag_cmd() { test_expect_success "retrieved object hashes back correctly" ' IPLDHASH2=$(cat ipld_obj_out | ipfs dag put --input-enc=dag-json -f dag-cbor) && - test "$IPLDCBORHASH" = "$IPLDHASH2" + test "$IPLDHASH" = "$IPLDHASH2" ' test_expect_success "add a normal file" ' @@ -141,7 +169,7 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - echo "{\"data\":\"CAISB2Zvb2JhcgoYBw==\",\"links\":[]}" > dag_get_pb_exp && + echo -n "{\"Data\":{\"/\":{\"bytes\":\"CAISB2Zvb2JhcgoYBw==\"}},\"Links\":[]}" > dag_get_pb_exp && test_cmp dag_get_pb_exp dag_get_pb_out ' @@ -150,7 +178,7 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - echo "{\"data\":\"CAISBGZvbwoYBA==\",\"links\":[]}" > cat_exp && + echo -n "{\"Data\":{\"/\":{\"bytes\":\"CAISBGZvbwoYBA==\"}},\"Links\":[]}" > cat_exp && test_cmp cat_exp cat_out ' @@ -179,13 +207,13 @@ test_dag_cmd() { ' test_expect_success "output looks correct" ' - EXPHASH="bafyriqf6pe376nnoh4ngsxmo7inplk35k5ucdfwvbc547vhp5mxx6wknllub3nyk44pvp37kgzrp2cotebl2onrqg5i7an4rcuu66cnj4ij4i" + EXPHASH="bafyriqdoys7rmgfkjg7jjdjdhaajt66vxjusd2ry6eims63xy5wwjtjgditq4z4f76tmnal5rccb4zg3o4zcud3utxulm7svoapq2du3xmnok" test $EXPHASH = $IPLDHASH ' test_expect_success "prepare dag-pb object" ' echo foo > test_file && - HASH=$(ipfs add -wq test_file | tail -n1) + HASH=$(ipfs add -wq test_file | tail -n1 | ipfs cid base32) ' test_expect_success "dag put with json dag-pb works" ' @@ -200,7 +228,7 @@ test_dag_cmd() { test_expect_success "dag put with raw dag-pb works" ' ipfs block get $HASH > pbraw && - cat pbraw | ipfs dag put --format=dag-pb --input-enc=raw > dag_put_out + cat pbraw | ipfs dag put --format=dag-pb --input-enc=dag-pb > dag_put_out ' test_expect_success "dag put with dag-pb works output looks good" ' @@ -287,10 +315,10 @@ test_dag_cmd() { test_expect_success "dag stat of simple IPLD object" ' ipfs dag stat $NESTED_HASH > actual_stat_inner_ipld_obj && - echo "Size: 15, NumBlocks: 1" > exp_stat_inner_ipld_obj && + echo "Size: 8, NumBlocks: 1" > exp_stat_inner_ipld_obj && test_cmp exp_stat_inner_ipld_obj actual_stat_inner_ipld_obj && ipfs dag stat $HASH > actual_stat_ipld_obj && - echo "Size: 61, NumBlocks: 2" > exp_stat_ipld_obj && + echo "Size: 54, NumBlocks: 2" > exp_stat_ipld_obj && test_cmp exp_stat_ipld_obj actual_stat_ipld_obj ' From fb25f6f21f2eaec883d78a735dee86526233f27a Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 16 Jul 2021 16:14:50 +1000 Subject: [PATCH 22/32] Temporarily use rvagg/dagjsonsort branch of go-ipld-prime --- go.mod | 5 ++--- go.sum | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2b510e746cb..f0971794ea2 100644 --- a/go.mod +++ b/go.mod @@ -61,8 +61,8 @@ require ( github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 github.com/ipld/go-car v0.3.1 - github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 - github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5 + github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 // indirect + github.com/ipld/go-ipld-prime v0.10.1-0.20210716061152-019d85c21ab4 github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 @@ -119,4 +119,3 @@ require ( ) go 1.14 -replace github.com/ipld/go-ipld-prime => ./../../ipld/go-ipld-prime/ \ No newline at end of file diff --git a/go.sum b/go.sum index 257fec11e4e..7147c29d70a 100644 --- a/go.sum +++ b/go.sum @@ -517,6 +517,8 @@ github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvB github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5 h1:hlKWs60jv1TN1WqYS4RueP9+MS2RRCFqj7CgJ+uDFQc= github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210716061152-019d85c21ab4 h1:P0193tWVYBZqARh+uaGbI1xvuJbuzQDSS6GWlxDnPuA= +github.com/ipld/go-ipld-prime v0.10.1-0.20210716061152-019d85c21ab4/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= From 1afd8e8ed08115ee480b0b3dccddbb9366be79cd Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 21 Jul 2021 13:42:58 +1000 Subject: [PATCH 23/32] Use current go-ipld-prime master with DAG-JSON sorting --- go.mod | 4 ++-- go.sum | 6 ++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f0971794ea2..f01bcad3613 100644 --- a/go.mod +++ b/go.mod @@ -61,8 +61,8 @@ require ( github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 github.com/ipld/go-car v0.3.1 - github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 // indirect - github.com/ipld/go-ipld-prime v0.10.1-0.20210716061152-019d85c21ab4 + github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 + github.com/ipld/go-ipld-prime v0.10.1-0.20210721023048-333be0f75cff github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 diff --git a/go.sum b/go.sum index 7147c29d70a..081040945af 100644 --- a/go.sum +++ b/go.sum @@ -515,10 +515,8 @@ github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/j github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5 h1:hlKWs60jv1TN1WqYS4RueP9+MS2RRCFqj7CgJ+uDFQc= -github.com/ipld/go-ipld-prime v0.10.1-0.20210701102120-b7347f196aa5/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.10.1-0.20210716061152-019d85c21ab4 h1:P0193tWVYBZqARh+uaGbI1xvuJbuzQDSS6GWlxDnPuA= -github.com/ipld/go-ipld-prime v0.10.1-0.20210716061152-019d85c21ab4/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210721023048-333be0f75cff h1:lq3003BnMXgz+sqwv2RBBalILy0AnY3TbXeyTBa34ww= +github.com/ipld/go-ipld-prime v0.10.1-0.20210721023048-333be0f75cff/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= From 65c8589aef446441423a2c719ca293ea808e53db Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 21 Jul 2021 16:31:18 +1000 Subject: [PATCH 24/32] Use mh.Set() for "hash" -> mhType --- core/commands/dag/put.go | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/core/commands/dag/put.go b/core/commands/dag/put.go index 0bbfbd9df63..9319f7f8a18 100644 --- a/core/commands/dag/put.go +++ b/core/commands/dag/put.go @@ -15,7 +15,6 @@ import ( files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" mc "github.com/multiformats/go-multicodec" - mh "github.com/multiformats/go-multihash" // Expected minimal set of available format/ienc codecs. _ "github.com/ipld/go-codec-dagpb" @@ -37,9 +36,6 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e hash, _ := req.Options["hash"].(string) dopin, _ := req.Options["pin"].(bool) - // mhType tells inputParser which hash should be used. Default otherwise is sha256 - mhType := uint64(mh.SHA2_256) - var icodec mc.Code if err := icodec.Set(ienc); err != nil { return err @@ -48,23 +44,18 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e if err := fcodec.Set(format); err != nil { return err } + var mhType mc.Code + if err := mhType.Set(hash); err != nil { + return err + } cidPrefix := cid.Prefix{ Version: 1, Codec: uint64(fcodec), - MhType: mhType, + MhType: uint64(mhType), MhLength: -1, } - if hash != "" { - var ok bool - mhType, ok = mh.Names[hash] - if !ok { - return fmt.Errorf("%s in not a valid multihash name", hash) - } - cidPrefix.MhType = mhType - } - decoder, err := multicodec.LookupDecoder(uint64(icodec)) if err != nil { return err From 5dd6196423e9aabd1ae3b304354f0c4bf3487bf2 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 21 Jul 2021 16:36:43 +1000 Subject: [PATCH 25/32] Make git plugin test work again --- plugin/plugins/git/git.go | 4 ++-- test/sharness/t0280-plugin-git.sh | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 3859cddaa79..5799017fa00 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -31,9 +31,9 @@ func (*gitPlugin) Version() string { } func (*gitPlugin) Init(_ *plugin.Environment) error { - // register a custom identifier for import of "zlib-incoded git objects." + // register a custom identifier in the reserved range for import of "zlib-encoded git objects." // TODO: give this a name. - multicodec.RegisterDecoder(uint64(0x10000000+mc.GitRaw), decodeZlibGit) + multicodec.RegisterDecoder(uint64(0x300000+mc.GitRaw), decodeZlibGit) return nil } diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index 4fee0b6aecb..d90df46bcce 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -17,19 +17,19 @@ test_expect_success "prepare test data" ' test_dag_git() { test_expect_success "add objects via dag put" ' - find objects -type f -exec ipfs dag put --format=git-raw --input-enc=268435576 {} \; -exec echo \; > hashes + find objects -type f -exec ipfs dag put --format=git-raw --input-enc=3145848 --hash=sha1 {} \; -exec echo -n \; > hashes ' - test_expect_failure "successfully get added objects" ' + test_expect_success "successfully get added objects" ' cat hashes | xargs -I {} ipfs dag get -- {} > /dev/null ' - test_expect_failure "path traversals work" ' - echo \"YmxvYiA3ACcsLnB5Zgo=\" > file1 && - ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/object/parents/0/tree/dir2/hash/f3/hash > out1 + test_expect_success "path traversals work" ' + echo -n "{\"/\":{\"bytes\":\"YmxvYiA3ACcsLnB5Zgo=\"}}" > file1 && + ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/Object/Parents/0/GitTree/1/Hash/0/Hash > out1 ' - test_expect_failure "outputs look correct" ' + test_expect_success "outputs look correct" ' test_cmp file1 out1 ' } From 62855629834092c0725f8e49a1d0cbb5f6d44014 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Wed, 21 Jul 2021 17:22:33 +1000 Subject: [PATCH 26/32] Default "hash" to sha2-256 --- core/commands/dag/dag.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 2e7aebb7daf..42a18336b5b 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -80,7 +80,7 @@ into an object of the specified format. cmds.StringOption("format", "f", "Format that the object will be added as.").WithDefault("dag-cbor"), cmds.StringOption("input-enc", "Format that the input object will be.").WithDefault("dag-json"), cmds.BoolOption("pin", "Pin this object when adding."), - cmds.StringOption("hash", "Hash function to use").WithDefault(""), + cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"), }, Run: dagPut, Type: OutputObject{}, From 705914a41eaf708a88a0f3aa428ef6faf2414d5c Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 22 Jul 2021 19:19:06 +1000 Subject: [PATCH 27/32] Use hex value in test to match code --- test/sharness/t0051-object.sh | 2 +- test/sharness/t0280-plugin-git.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sharness/t0051-object.sh b/test/sharness/t0051-object.sh index 82e746ddec8..95de4fda758 100755 --- a/test/sharness/t0051-object.sh +++ b/test/sharness/t0051-object.sh @@ -130,7 +130,7 @@ test_object_cmd() { ' test_expect_failure "'ipfs object put --inputenc=xml' output looks good" ' - echo "added $HASH" >expected && + echo "added $HASH\n" >expected && test_cmp expected actual ' diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index d90df46bcce..c3b3a4a49be 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -17,7 +17,7 @@ test_expect_success "prepare test data" ' test_dag_git() { test_expect_success "add objects via dag put" ' - find objects -type f -exec ipfs dag put --format=git-raw --input-enc=3145848 --hash=sha1 {} \; -exec echo -n \; > hashes + find objects -type f -exec ipfs dag put --format=git-raw --input-enc=0x300078 --hash=sha1 {} \; -exec echo -n \; > hashes ' test_expect_success "successfully get added objects" ' From 963d53b8810287bb2e2b7dbc96f34d81858e77cf Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Thu, 22 Jul 2021 21:57:14 +1000 Subject: [PATCH 28/32] Update to latest go-ipld-git, reset sharness test paths to original --- go.mod | 2 +- go.sum | 4 ++-- test/sharness/t0280-plugin-git.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f01bcad3613..b25acfe8070 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc + github.com/ipfs/go-ipld-git v0.0.4-0.20210722115044-0436bc7c4a27 github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 diff --git a/go.sum b/go.sum index 081040945af..c129502de90 100644 --- a/go.sum +++ b/go.sum @@ -449,8 +449,8 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc h1:57luC/Oc36sZBp4BAJ74MJGsAULBeQ5NLLRpTL6YJwI= -github.com/ipfs/go-ipld-git v0.0.4-0.20210326054912-6be33ddcbecc/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= +github.com/ipfs/go-ipld-git v0.0.4-0.20210722115044-0436bc7c4a27 h1:ziq9+3WwmgmSRmyquYdIXECGHUk9B+hEshIGMV1Kkp4= +github.com/ipfs/go-ipld-git v0.0.4-0.20210722115044-0436bc7c4a27/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= diff --git a/test/sharness/t0280-plugin-git.sh b/test/sharness/t0280-plugin-git.sh index c3b3a4a49be..c9720a0dd6f 100755 --- a/test/sharness/t0280-plugin-git.sh +++ b/test/sharness/t0280-plugin-git.sh @@ -26,7 +26,7 @@ test_dag_git() { test_expect_success "path traversals work" ' echo -n "{\"/\":{\"bytes\":\"YmxvYiA3ACcsLnB5Zgo=\"}}" > file1 && - ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/Object/Parents/0/GitTree/1/Hash/0/Hash > out1 + ipfs dag get baf4bcfhzi72pcj5cc4ocz7igcduubuu7aa3cddi/object/parents/0/tree/dir2/hash/f3/hash > out1 ' test_expect_success "outputs look correct" ' From 6522df191b29972c1d7ce239d303888d2656a9d7 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Fri, 23 Jul 2021 19:29:46 +1000 Subject: [PATCH 29/32] Use mc.ReservedStart from new go-multiformats --- go.mod | 2 +- go.sum | 4 ++-- plugin/plugins/git/git.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index b25acfe8070..98097f36351 100644 --- a/go.mod +++ b/go.mod @@ -97,7 +97,7 @@ require ( github.com/multiformats/go-multiaddr v0.3.1 github.com/multiformats/go-multiaddr-dns v0.2.0 github.com/multiformats/go-multibase v0.0.3 - github.com/multiformats/go-multicodec v0.2.1-0.20210718225951-a237c4436b76 + github.com/multiformats/go-multicodec v0.2.1-0.20210723011827-52a3ac546597 github.com/multiformats/go-multihash v0.0.15 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index c129502de90..7b33b955064 100644 --- a/go.sum +++ b/go.sum @@ -958,8 +958,8 @@ github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5 github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3 h1:l/B6bJDQjvQ5G52jw4QGSYeOTZoAwIO77RblWplfIqk= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multicodec v0.2.1-0.20210718225951-a237c4436b76 h1:VD/M8CCDR5uOoNpeCSejcHp3MY7XaQYqLr3ANrDDRxQ= -github.com/multiformats/go-multicodec v0.2.1-0.20210718225951-a237c4436b76/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= +github.com/multiformats/go-multicodec v0.2.1-0.20210723011827-52a3ac546597 h1:ULcqjrvbMRf7TiRiXlmXprupBxpeUkL8q2N8Xq+bNi4= +github.com/multiformats/go-multicodec v0.2.1-0.20210723011827-52a3ac546597/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= diff --git a/plugin/plugins/git/git.go b/plugin/plugins/git/git.go index 5799017fa00..e1d38e5b7d0 100644 --- a/plugin/plugins/git/git.go +++ b/plugin/plugins/git/git.go @@ -33,7 +33,7 @@ func (*gitPlugin) Version() string { func (*gitPlugin) Init(_ *plugin.Environment) error { // register a custom identifier in the reserved range for import of "zlib-encoded git objects." // TODO: give this a name. - multicodec.RegisterDecoder(uint64(0x300000+mc.GitRaw), decodeZlibGit) + multicodec.RegisterDecoder(uint64(mc.ReservedStart+mc.GitRaw), decodeZlibGit) return nil } From fc9062bed12d1209dd20e3661d7494e978cb2d61 Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Mon, 26 Jul 2021 17:26:57 +1000 Subject: [PATCH 30/32] Update deps --- go.mod | 6 +++--- go.sum | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 98097f36351..bb23be27cc0 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 github.com/ipfs/go-ipld-cbor v0.0.5 github.com/ipfs/go-ipld-format v0.2.0 - github.com/ipfs/go-ipld-git v0.0.4-0.20210722115044-0436bc7c4a27 + github.com/ipfs/go-ipld-git v0.0.4-0.20210726071918-f3dcf6c6f96e github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.4 @@ -54,7 +54,7 @@ require ( github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-metrics-prometheus v0.0.2 github.com/ipfs/go-mfs v0.1.2 - github.com/ipfs/go-path v0.0.10-0.20210421213242-802a897dfcd8 + github.com/ipfs/go-path v0.0.10-0.20210726072238-b450033b62d9 github.com/ipfs/go-pinning-service-http-client v0.1.0 github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-unixfsnode v1.1.1 @@ -62,7 +62,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 github.com/ipld/go-car v0.3.1 github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 - github.com/ipld/go-ipld-prime v0.10.1-0.20210721023048-333be0f75cff + github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39 github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 diff --git a/go.sum b/go.sum index 7b33b955064..13bb2185bf5 100644 --- a/go.sum +++ b/go.sum @@ -449,8 +449,8 @@ github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dC github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-git v0.0.4-0.20210722115044-0436bc7c4a27 h1:ziq9+3WwmgmSRmyquYdIXECGHUk9B+hEshIGMV1Kkp4= -github.com/ipfs/go-ipld-git v0.0.4-0.20210722115044-0436bc7c4a27/go.mod h1:jtK6vhzvHtNm2YhKRxKMR8IQJAoZRtYoWqf8RnDeaIA= +github.com/ipfs/go-ipld-git v0.0.4-0.20210726071918-f3dcf6c6f96e h1:nIn/bK6M+x9YjgDIs0JU9BexgFlOiSXA5MnVIOK3yhM= +github.com/ipfs/go-ipld-git v0.0.4-0.20210726071918-f3dcf6c6f96e/go.mod h1:DHEYwGO2tc27/7VVRtNqFvJBFbLufPW75eGkvYJgNcU= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0 h1:PXjRvJkxqQ2EbwviF5GEURYqvhS/ulee/ZdVsVgVtHM= github.com/ipfs/go-ipld-legacy v0.0.0-20210325015318-9799f2cffab0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= @@ -485,8 +485,8 @@ github.com/ipfs/go-mfs v0.1.2 h1:DlelNSmH+yz/Riy0RjPKlooPg0KML4lXGdLw7uZkfAg= github.com/ipfs/go-mfs v0.1.2/go.mod h1:T1QBiZPEpkPLzDqEJLNnbK55BVKVlNi2a+gVm4diFo0= github.com/ipfs/go-path v0.0.7/go.mod h1:6KTKmeRnBXgqrTvzFrPV3CamxcgvXX/4z79tfAd2Sno= github.com/ipfs/go-path v0.0.10-0.20210405201800-40f1060226f7/go.mod h1:g8egwymo/MjfPUu/ojghk4YQKwjn6MGO+UUv4eUw08M= -github.com/ipfs/go-path v0.0.10-0.20210421213242-802a897dfcd8 h1:YT89FH52ynHRCZJIM95z+/4fz0rQ7nFPFLRufOzm9AE= -github.com/ipfs/go-path v0.0.10-0.20210421213242-802a897dfcd8/go.mod h1:YMCaFyukoM1bSaPm1eL9Q6IPr0z4WytNeVKlBSm8a0c= +github.com/ipfs/go-path v0.0.10-0.20210726072238-b450033b62d9 h1:GEptmVxRDOWObHiqZPbk9PyLlAwyZyCrBc4hu3oGqeI= +github.com/ipfs/go-path v0.0.10-0.20210726072238-b450033b62d9/go.mod h1:grgssIrSAXeXl3OQ7PfI3yMPbD6v84HrvLjkkFuvIdc= github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= @@ -513,10 +513,9 @@ github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 h1:m/ZZEoOds github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2/go.mod h1:GMLfso6KSkYJlIbd2cGKdGMe/hM5/IukeXRQ+u6zTrQ= github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.10.1-0.20210721023048-333be0f75cff h1:lq3003BnMXgz+sqwv2RBBalILy0AnY3TbXeyTBa34ww= -github.com/ipld/go-ipld-prime v0.10.1-0.20210721023048-333be0f75cff/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39 h1:70/42k1tU7ZHuyjFDd7vbd6dgnMyFnC0fe8LKeF9ZPg= +github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= From 4f0c15f6d2ad5ff0f6b1d5d7b6730f5f13ca3f2a Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 27 Jul 2021 09:08:21 +1000 Subject: [PATCH 31/32] Node dag-cbor sorting TODO --- test/sharness/t0053-dag.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 826a2eda132..f14fe25b15e 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -21,6 +21,8 @@ test_expect_success "make a few test files" ' HASH4=$(ipfs add --pin=false -q file4) ' +# TODO: revert this ordering when https://github.com/ipld/go-ipld-prime/pull/204 is merged +# and we get dag-cbor map sorting test_expect_success "make an ipld object in json" ' printf "{\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"hello\":\"world\",\"magic\":{\"/\":\"%s\"},\"subNode\":{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}}" $HASH1 $HASH2 $HASH3 > ipld_object ' From c3b6d26b38a2f46caad2e11a0ac90bbe59a9844f Mon Sep 17 00:00:00 2001 From: Rod Vagg Date: Tue, 27 Jul 2021 13:20:24 +1000 Subject: [PATCH 32/32] Update go-ipld-prime, revert tests to sorted-cbor forms --- go.mod | 2 +- go.sum | 3 ++- test/sharness/t0053-dag.sh | 27 +++++++++++++-------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index bb23be27cc0..a123bfe0de3 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.4.1-0.20210410030100-56c2e678cb62 github.com/ipld/go-car v0.3.1 github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2 - github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39 + github.com/ipld/go-ipld-prime v0.10.1-0.20210727022528-083c39518830 github.com/jbenet/go-is-domain v1.0.5 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 diff --git a/go.sum b/go.sum index 13bb2185bf5..959a03ec19f 100644 --- a/go.sum +++ b/go.sum @@ -514,8 +514,9 @@ github.com/ipld/go-codec-dagpb v1.2.1-0.20210405170603-d0b86f7623c2/go.mod h1:GM github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210402181957-7406578571d1/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39 h1:70/42k1tU7ZHuyjFDd7vbd6dgnMyFnC0fe8LKeF9ZPg= github.com/ipld/go-ipld-prime v0.10.1-0.20210723093545-ebe39c7bab39/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.10.1-0.20210727022528-083c39518830 h1:tQyWv1pyxNIFe2hPczHGITClSQRJlW6dLIcEvkdSRgY= +github.com/ipld/go-ipld-prime v0.10.1-0.20210727022528-083c39518830/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index f14fe25b15e..b95e853cd8b 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -21,10 +21,8 @@ test_expect_success "make a few test files" ' HASH4=$(ipfs add --pin=false -q file4) ' -# TODO: revert this ordering when https://github.com/ipld/go-ipld-prime/pull/204 is merged -# and we get dag-cbor map sorting test_expect_success "make an ipld object in json" ' - printf "{\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"hello\":\"world\",\"magic\":{\"/\":\"%s\"},\"subNode\":{\"beep\":[0,\"bop\"],\"dict\":\"ionary\"}}" $HASH1 $HASH2 $HASH3 > ipld_object + printf "{\"hello\":\"world\",\"cats\":[{\"/\":\"%s\"},{\"water\":{\"/\":\"%s\"}}],\"magic\":{\"/\":\"%s\"},\"sub\":{\"dict\":\"ionary\",\"beep\":[0,\"bop\"]}}" $HASH1 $HASH2 $HASH3 > ipld_object ' test_expect_success "make the same ipld object in dag-cbor, dag-json and dag-pb" ' @@ -39,7 +37,8 @@ test_dag_cmd() { ' test_expect_success "CID looks correct" ' - EXPHASH="bafyreihim5n3etxp5om74fdpriqtys5sdmgbd5ix2oil7u2g2qfdpdlgie" + EXPHASH="bafyreiblwimnjbqcdoeafiobk6q27jcw64ew7n2fmmhdpldd63edmjecde" + echo $IPLDHASH > ipld_hash && \ test $EXPHASH = $IPLDHASH ' @@ -48,7 +47,7 @@ test_dag_cmd() { ' test_expect_success "CID looks correct" ' - EXPHASH="bafyreihim5n3etxp5om74fdpriqtys5sdmgbd5ix2oil7u2g2qfdpdlgie" + EXPHASH="bafyreiblwimnjbqcdoeafiobk6q27jcw64ew7n2fmmhdpldd63edmjecde" test $EXPHASH = $IPLDHASH ' @@ -109,10 +108,10 @@ test_dag_cmd() { test_expect_success "resolving sub-objects works" ' ipfs dag get $IPLDHASH/hello > sub1 && - ipfs dag get $IPLDHASH/subNode > sub2 && - ipfs dag get $IPLDHASH/subNode/beep > sub3 && - ipfs dag get $IPLDHASH/subNode/beep/0 > sub4 && - ipfs dag get $IPLDHASH/subNode/beep/1 > sub5 + ipfs dag get $IPLDHASH/sub > sub2 && + ipfs dag get $IPLDHASH/sub/beep > sub3 && + ipfs dag get $IPLDHASH/sub/beep/0 > sub4 && + ipfs dag get $IPLDHASH/sub/beep/1 > sub5 ' test_expect_success "sub-objects look right" ' @@ -184,9 +183,9 @@ test_dag_cmd() { test_cmp cat_exp cat_out ' - test_expect_success "non-canonical cbor input is normalized" ' + test_expect_success "non-canonical dag-cbor input is normalized" ' HASH=$(cat ../t0053-dag-data/non-canon.cbor | ipfs dag put --format=dag-cbor --input-enc=dag-cbor) && - test $HASH = "bafyreicfngo3hexoxbbwsm64apbydw5fyeruyrfwiyxlj3f4wuzdyztd4m" || + test $HASH = "bafyreiawx7ona7oa2ptcoh6vwq4q6bmd7x2ibtkykld327bgb7t73ayrqm" || test_fsh echo $HASH ' @@ -204,12 +203,12 @@ test_dag_cmd() { ipfs refs -r --timeout=2s $PINHASH > /dev/null ' - test_expect_success "can add an ipld object with sha3 hash" ' - IPLDHASH=$(cat ipld_object | ipfs dag put --hash sha3) + test_expect_success "can add an ipld object with sha3-512 hash" ' + IPLDHASH=$(cat ipld_object | ipfs dag put --hash sha3-512) ' test_expect_success "output looks correct" ' - EXPHASH="bafyriqdoys7rmgfkjg7jjdjdhaajt66vxjusd2ry6eims63xy5wwjtjgditq4z4f76tmnal5rccb4zg3o4zcud3utxulm7svoapq2du3xmnok" + EXPHASH="bafyriqforjma7y7akqz7nhuu73r6liggj5zhkbfiqgicywe3fgkna2ijlhod2af3ue7doj56tlzt5hh6iu5esafc4msr3sd53jol5m2o25ucy" test $EXPHASH = $IPLDHASH '