From 9d3ff4f51dd6fcaf5ab2b27eee957f749c16a764 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 3 Feb 2020 17:35:51 +0100 Subject: [PATCH] Work with Multihashes directly This will be breaking for anyone using filestore who has not migrated the underlying storage and coverted CIDs to raw multihashes. It keeps the outside interfaces intact except for `ListRes` which includes a multihash rather than a Cid. --- filestore.go | 4 ++-- fsrefstore.go | 49 +++++++++++++++++++++++++++++-------------------- go.mod | 6 +++++- go.sum | 36 ++++++++++++++++++++++-------------- util.go | 41 +++++++++++++++++++++-------------------- 5 files changed, 79 insertions(+), 57 deletions(-) diff --git a/filestore.go b/filestore.go index be4d954..a9c36c5 100644 --- a/filestore.go +++ b/filestore.go @@ -19,7 +19,7 @@ import ( logging "github.com/ipfs/go-log" ) -var log = logging.Logger("filestore") +var logger = logging.Logger("filestore") var ErrFilestoreNotEnabled = errors.New("filestore is not enabled, see https://git.io/vNItf") var ErrUrlstoreNotEnabled = errors.New("urlstore is not enabled") @@ -86,7 +86,7 @@ func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { // cant query leveldb concurrently b, err := f.fm.AllKeysChan(ctx) if err != nil { - log.Error("error querying filestore: ", err) + logger.Error("error querying filestore: ", err) return } diff --git a/fsrefstore.go b/fsrefstore.go index 19927e0..bc183fc 100644 --- a/fsrefstore.go +++ b/fsrefstore.go @@ -19,6 +19,7 @@ import ( blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" posinfo "github.com/ipfs/go-ipfs-posinfo" + mh "github.com/multiformats/go-multihash" ) // FilestorePrefix identifies the key prefix for FileManager blocks. @@ -60,6 +61,8 @@ func NewFileManager(ds ds.Batching, root string) *FileManager { // AllKeysChan returns a channel from which to read the keys stored in // the FileManager. If the given context is cancelled the channel will be // closed. +// +// All CIDs returned are of type Raw. func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { q := dsq.Query{KeysOnly: true} @@ -78,14 +81,14 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { } k := ds.RawKey(v.Key) - c, err := dshelp.DsKeyToCid(k) + mhash, err := dshelp.DsKeyToMultihash(k) if err != nil { - log.Errorf("decoding cid from filestore: %s", err) + logger.Errorf("decoding cid from filestore: %s", err) continue } select { - case out <- c: + case out <- cid.NewCidV1(cid.Raw, mhash): case <-ctx.Done(): return } @@ -98,7 +101,7 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) { // DeleteBlock deletes the reference-block from the underlying // datastore. It does not touch the referenced data. func (f *FileManager) DeleteBlock(c cid.Cid) error { - err := f.ds.Delete(dshelp.CidToDsKey(c)) + err := f.ds.Delete(dshelp.MultihashToDsKey(c.Hash())) if err == ds.ErrNotFound { return blockstore.ErrNotFound } @@ -110,11 +113,11 @@ func (f *FileManager) DeleteBlock(c cid.Cid) error { // block from the datastore. The second step uses the stored // path and offsets to read the raw block data directly from disk. func (f *FileManager) Get(c cid.Cid) (blocks.Block, error) { - dobj, err := f.getDataObj(c) + dobj, err := f.getDataObj(c.Hash()) if err != nil { return nil, err } - out, err := f.readDataObj(c, dobj) + out, err := f.readDataObj(c.Hash(), dobj) if err != nil { return nil, err } @@ -127,22 +130,22 @@ func (f *FileManager) Get(c cid.Cid) (blocks.Block, error) { // This method may successfully return the size even if returning the block // would fail because the associated file is no longer available. func (f *FileManager) GetSize(c cid.Cid) (int, error) { - dobj, err := f.getDataObj(c) + dobj, err := f.getDataObj(c.Hash()) if err != nil { return -1, err } return int(dobj.GetSize_()), nil } -func (f *FileManager) readDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { if IsURL(d.GetFilePath()) { - return f.readURLDataObj(c, d) + return f.readURLDataObj(m, d) } - return f.readFileDataObj(c, d) + return f.readFileDataObj(m, d) } -func (f *FileManager) getDataObj(c cid.Cid) (*pb.DataObj, error) { - o, err := f.ds.Get(dshelp.CidToDsKey(c)) +func (f *FileManager) getDataObj(m mh.Multihash) (*pb.DataObj, error) { + o, err := f.ds.Get(dshelp.MultihashToDsKey(m)) switch err { case ds.ErrNotFound: return nil, blockstore.ErrNotFound @@ -164,7 +167,7 @@ func unmarshalDataObj(data []byte) (*pb.DataObj, error) { return &dobj, nil } -func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readFileDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { if !f.AllowFiles { return nil, ErrFilestoreNotEnabled } @@ -193,12 +196,15 @@ func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) return nil, &CorruptReferenceError{StatusFileError, err} } - outcid, err := c.Prefix().Sum(outbuf) + // Work with CIDs for this, as they are a nice wrapper and things + // will not break if multihashes underlying types change. + origCid := cid.NewCidV1(cid.Raw, m) + outcid, err := origCid.Prefix().Sum(outbuf) if err != nil { return nil, err } - if !c.Equals(outcid) { + if !origCid.Equals(outcid) { return nil, &CorruptReferenceError{StatusFileChanged, fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} } @@ -207,7 +213,7 @@ func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) } // reads and verifies the block from URL -func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { +func (f *FileManager) readURLDataObj(m mh.Multihash, d *pb.DataObj) ([]byte, error) { if !f.AllowUrls { return nil, ErrUrlstoreNotEnabled } @@ -237,12 +243,15 @@ func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { } res.Body.Close() - outcid, err := c.Prefix().Sum(outbuf) + // Work with CIDs for this, as they are a nice wrapper and things + // will not break if multihashes underlying types change. + origCid := cid.NewCidV1(cid.Raw, m) + outcid, err := origCid.Prefix().Sum(outbuf) if err != nil { return nil, err } - if !c.Equals(outcid) { + if !origCid.Equals(outcid) { return nil, &CorruptReferenceError{StatusFileChanged, fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())} } @@ -255,7 +264,7 @@ func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) { func (f *FileManager) Has(c cid.Cid) (bool, error) { // NOTE: interesting thing to consider. Has doesnt validate the data. // So the data on disk could be invalid, and we could think we have it. - dsk := dshelp.CidToDsKey(c) + dsk := dshelp.MultihashToDsKey(c.Hash()) return f.ds.Has(dsk) } @@ -300,7 +309,7 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error { return err } - return to.Put(dshelp.CidToDsKey(b.Cid()), data) + return to.Put(dshelp.MultihashToDsKey(b.Cid().Hash()), data) } // PutMany is like Put() but takes a slice of blocks instead, diff --git a/go.mod b/go.mod index ac9659d..61f4940 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,16 @@ go 1.12 require ( github.com/gogo/protobuf v1.3.1 + github.com/golang/protobuf v1.3.2 // indirect github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-cid v0.0.5 github.com/ipfs/go-datastore v0.4.1 github.com/ipfs/go-ipfs-blockstore v0.1.3 github.com/ipfs/go-ipfs-ds-help v0.0.1 github.com/ipfs/go-ipfs-posinfo v0.0.1 - github.com/ipfs/go-log v0.0.1 + github.com/ipfs/go-log v1.0.1 github.com/ipfs/go-merkledag v0.3.1 + github.com/multiformats/go-multihash v0.0.10 + golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect + gopkg.in/yaml.v2 v2.2.4 // indirect ) diff --git a/go.sum b/go.sum index 5f596ef..44b3cbe 100644 --- a/go.sum +++ b/go.sum @@ -29,14 +29,14 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJY github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.0 h1:G8O7TerXerS4F6sx9OV7/nRfJdnXgHZu/S/7F2SN+UE= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -49,8 +49,6 @@ github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJ github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -99,10 +97,8 @@ github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xL github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0 h1:V1GZorHFUIB6YgTJQdq7mcaIpUfCM3fCyVi+MTo9O88= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= -github.com/ipfs/go-ipfs-blockstore v0.1.1 h1:+PAFREAlSxLs9IEtrRcnJ/DqWkGlDa+l547WFZnohNw= -github.com/ipfs/go-ipfs-blockstore v0.1.1/go.mod h1:8gZOgIN5e+Xdg2YSGdwTTRbguSVjYyosIDRQCY8E9QM= -github.com/ipfs/go-ipfs-blockstore v0.1.3 h1:Nc/Uwc8KZgo5PsMUZT/Zt7zc0u/s+b/3c64ChQNttHc= -github.com/ipfs/go-ipfs-blockstore v0.1.3/go.mod h1:iNWVBoSQ7eMcaGo8+L3pKZABGTdWcqj1/hpoUu5bDps= +github.com/ipfs/go-ipfs-blockstore v0.1.2 h1:hmvFNsFjfrIVm5Dgwumsw+MVWIjK4CaPrM/W7fKV5YY= +github.com/ipfs/go-ipfs-blockstore v0.1.2/go.mod h1:2VcQdtvLIDncMc0ciBZygpDNfwrcJfOEC0LSXyG2AVc= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -110,6 +106,8 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1 h1:QBg+Ts2zgeemK/dB0saiF/ykzRGgfoFMT90Rzo0OnVU= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= +github.com/ipfs/go-ipfs-ds-help v0.1.0 h1:/MfUnPgrtRCJvLdOiAgaODJD/BtV1CpfqQ53eqhWXqM= +github.com/ipfs/go-ipfs-ds-help v0.1.0/go.mod h1:zjN0aB3d6VyzJTSvcylSYxo+LSR/ELEfNnQM8p/vwc8= github.com/ipfs/go-ipfs-exchange-interface v0.0.1 h1:LJXIo9W7CAmugqI+uofioIpRb6rY30GUu7G6LUfpMvM= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C81I1NSHW1FxGew= @@ -122,8 +120,6 @@ github.com/ipfs/go-ipfs-routing v0.1.0 h1:gAJTT1cEeeLj6/DlLX6t+NxD9fQe2ymTO6qWRD github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv50= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipld-cbor v0.0.2 h1:amzFztBQQQ69UA5+f7JRfoXF/z2l//MGfEDHVkS20+s= -github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-format v0.0.1 h1:HCu4eB/Gh+KD/Q0M8u888RFkorTWNIL3da4oc5dwc80= @@ -132,10 +128,10 @@ github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2 github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= -github.com/ipfs/go-merkledag v0.1.0 h1:CAEXjRFEDPvealQj3TgEjV1IJckwjvmxAqtq5QSXJrg= -github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= -github.com/ipfs/go-merkledag v0.2.4 h1:ZSHQSe9BENfixUjT+MaLeHEeZGxrZQfgo3KT3SLosF8= -github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-log v1.0.1 h1:5lIEEOQTk/vd1WuPFBRqz2mcp+5G1fMVcW+Ib/H5Hfo= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= +github.com/ipfs/go-log/v2 v2.0.1 h1:mnR9XFltezAtO8A6tj5U7nKkRzhEQNEw/wT11U2HhPM= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ= github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= @@ -324,6 +320,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436 h1:qOpVTI+BrstcjTZLm2Yz/3sOnqkzj3FQoh0g+E5s3Gc= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= @@ -337,6 +335,12 @@ github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvX github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 h1:E9S12nwJwEOXe2d6gT6qxdvqMnNq+VnSsKPgm2ZZNds= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -354,6 +358,8 @@ golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -384,3 +390,5 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/util.go b/util.go index bfb240c..de1e3bd 100644 --- a/util.go +++ b/util.go @@ -11,6 +11,7 @@ import ( dsq "github.com/ipfs/go-datastore/query" blockstore "github.com/ipfs/go-ipfs-blockstore" dshelp "github.com/ipfs/go-ipfs-ds-help" + mh "github.com/multiformats/go-multihash" ) // Status is used to identify the state of the block data referenced @@ -60,7 +61,7 @@ func (s Status) Format() string { type ListRes struct { Status Status ErrorMsg string - Key cid.Cid + Key mh.Multihash FilePath string Offset uint64 Size uint64 @@ -72,12 +73,12 @@ func (r *ListRes) FormatLong(enc func(cid.Cid) string) string { enc = (cid.Cid).String } switch { - case !r.Key.Defined(): + case r.Key == nil: return "" case r.FilePath == "": return r.Key.String() default: - return fmt.Sprintf("%-50s %6d %s %d", enc(r.Key), r.Size, r.FilePath, r.Offset) + return fmt.Sprintf("%-50s %6d %s %d", enc(cid.NewCidV1(cid.Raw, r.Key)), r.Size, r.FilePath, r.Offset) } } @@ -86,7 +87,7 @@ func (r *ListRes) FormatLong(enc func(cid.Cid) string) string { // List does not verify that the reference is valid or whether the // raw data is accesible. See Verify(). func List(fs *Filestore, key cid.Cid) *ListRes { - return list(fs, false, key) + return list(fs, false, key.Hash()) } // ListAll returns a function as an iterator which, once invoked, returns @@ -105,7 +106,7 @@ func ListAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) { // Verify makes sure that the reference is valid and the block data can be // read. func Verify(fs *Filestore, key cid.Cid) *ListRes { - return list(fs, true, key) + return list(fs, true, key.Hash()) } // VerifyAll returns a function as an iterator which, once invoked, @@ -119,7 +120,7 @@ func VerifyAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) { return listAll(fs, true) } -func list(fs *Filestore, verify bool, key cid.Cid) *ListRes { +func list(fs *Filestore, verify bool, key mh.Multihash) *ListRes { dobj, err := fs.fm.getDataObj(key) if err != nil { return mkListRes(key, nil, err) @@ -138,34 +139,34 @@ func listAll(fs *Filestore, verify bool) (func() *ListRes, error) { } return func() *ListRes { - cid, dobj, err := next(qr) + mhash, dobj, err := next(qr) if dobj == nil && err == nil { return nil } else if err == nil && verify { - _, err = fs.fm.readDataObj(cid, dobj) + _, err = fs.fm.readDataObj(mhash, dobj) } - return mkListRes(cid, dobj, err) + return mkListRes(mhash, dobj, err) }, nil } -func next(qr dsq.Results) (cid.Cid, *pb.DataObj, error) { +func next(qr dsq.Results) (mh.Multihash, *pb.DataObj, error) { v, ok := qr.NextSync() if !ok { - return cid.Cid{}, nil, nil + return nil, nil, nil } k := ds.RawKey(v.Key) - c, err := dshelp.DsKeyToCid(k) + mhash, err := dshelp.DsKeyToMultihash(k) if err != nil { - return cid.Cid{}, nil, fmt.Errorf("decoding cid from filestore: %s", err) + return nil, nil, fmt.Errorf("decoding multihash from filestore: %s", err) } dobj, err := unmarshalDataObj(v.Value) if err != nil { - return c, nil, err + return mhash, nil, err } - return c, dobj, nil + return mhash, dobj, nil } func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) { @@ -206,9 +207,9 @@ func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) { } v := entries[i] i++ - // attempt to convert the datastore key to a CID, + // attempt to convert the datastore key to a Multihash, // store the error but don't use it yet - cid, keyErr := dshelp.DsKeyToCid(ds.RawKey(v.dsKey)) + cid, keyErr := dshelp.DsKeyToMultihash(ds.RawKey(v.dsKey)) // first if they listRes already had an error return that error if v.err != nil { return mkListRes(cid, nil, v.err) @@ -255,7 +256,7 @@ func (l listEntries) Less(i, j int) bool { return l[i].filePath < l[j].filePath } -func mkListRes(c cid.Cid, d *pb.DataObj, err error) *ListRes { +func mkListRes(m mh.Multihash, d *pb.DataObj, err error) *ListRes { status := StatusOk errorMsg := "" if err != nil { @@ -272,14 +273,14 @@ func mkListRes(c cid.Cid, d *pb.DataObj, err error) *ListRes { return &ListRes{ Status: status, ErrorMsg: errorMsg, - Key: c, + Key: m, } } return &ListRes{ Status: status, ErrorMsg: errorMsg, - Key: c, + Key: m, FilePath: d.FilePath, Size: d.Size_, Offset: d.Offset,