From 877a66bf025296cf8bba019f16728ff0997dda50 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 3 Feb 2023 15:39:12 +0100 Subject: [PATCH] ipsl: use blocks interface Will be fine performance wise once #57 is fixed. --- ipsl/compile.go | 5 +++-- ipsl/helpers/helpers.go | 35 +++++------------------------------ ipsl/helpers/helpers_test.go | 31 ++++++++++++++++++++++++------- ipsl/ipsl.go | 8 +++++--- ipsl/ipsl_test.go | 3 ++- ipsl/unixfs/unixfs.go | 12 ++++++------ ipsl/unixfs/unixfs_test.go | 16 ++++++++-------- 7 files changed, 53 insertions(+), 57 deletions(-) diff --git a/ipsl/compile.go b/ipsl/compile.go index 3b7d9d86d..5eca4b68f 100644 --- a/ipsl/compile.go +++ b/ipsl/compile.go @@ -8,6 +8,7 @@ import ( "sync" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" ) type UnreadableRuneReader interface { @@ -338,8 +339,8 @@ type traversalScopeNode struct { scopeNode } -func (n traversalScopeNode) Traverse(c cid.Cid, b []byte) ([]CidTraversalPair, error) { - r, err := n.scopeNode.result.(Traversal).Traverse(c, b) +func (n traversalScopeNode) Traverse(b blocks.Block) ([]CidTraversalPair, error) { + r, err := n.scopeNode.result.(Traversal).Traverse(b) if err != nil { return nil, err } diff --git a/ipsl/helpers/helpers.go b/ipsl/helpers/helpers.go index 18007bd87..70bc0865c 100644 --- a/ipsl/helpers/helpers.go +++ b/ipsl/helpers/helpers.go @@ -8,43 +8,18 @@ import ( "errors" "fmt" - "github.com/ipfs/go-libipfs/ipsl" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/ipsl" ) -var _ ByteBlockGetter = BlockGetterToByteBlockGetter{} - -// BlockGetterToByteBlockGetter implements [ByteBlockGetter] given an [blockservice.BlockGetter]. -// This prevent the optimizations [ByteBlockGetter] is trying to do, so you should ideally not use this. -type BlockGetterToByteBlockGetter struct { - BlockGetter blockservice.BlockGetter -} - -func (bgtbbg BlockGetterToByteBlockGetter) GetBlock(ctx context.Context, c cid.Cid) ([]byte, error) { - b, err := bgtbbg.BlockGetter.GetBlock(ctx, c) - if err != nil { - return nil, err - } - - return b.RawData(), nil -} - -// ByteBlockGetter is like [blockservice.BlockGetter] but it does not wrap in an extra block object. -// This is to avoid making too much garbage, passing blocks interface objects arround is moves stuff on the heap. -type ByteBlockGetter interface { - GetBlock(ctx context.Context, c cid.Cid) ([]byte, error) - - // TODO: Add GetBlocks when that is needed. -} - var ErrDepthLimitReached = errors.New("safety depth limit reached") // SyncDFS perform a synchronous recursive depth-first-search. // It will return [ErrDepthLimitReached] when the safetyDepthLimit is reached. // It will wrap errors returned by the call back, so use [errors.Is] to test them. -func SyncDFS(ctx context.Context, c cid.Cid, t ipsl.Traversal, getter ByteBlockGetter, safetyDepthLimit uint, callBack func(cid.Cid, []byte) error) error { +func SyncDFS(ctx context.Context, c cid.Cid, t ipsl.Traversal, getter blockservice.BlockGetter, safetyDepthLimit uint, callBack func(blocks.Block) error) error { if safetyDepthLimit == 0 { return ErrDepthLimitReached } @@ -55,12 +30,12 @@ func SyncDFS(ctx context.Context, c cid.Cid, t ipsl.Traversal, getter ByteBlockG return fmt.Errorf("GetBlock: %w", err) } - err = callBack(c, block) + err = callBack(block) if err != nil { return fmt.Errorf("callBack: %w", err) } - pairs, err := t.Traverse(c, block) + pairs, err := t.Traverse(block) if err != nil { return fmt.Errorf("Traversal.Traverse: %w", err) } diff --git a/ipsl/helpers/helpers_test.go b/ipsl/helpers/helpers_test.go index b9948b3d6..9e3a677a7 100644 --- a/ipsl/helpers/helpers_test.go +++ b/ipsl/helpers/helpers_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipfs/go-libipfs/ipsl" . "github.com/ipfs/go-libipfs/ipsl/helpers" "github.com/multiformats/go-multihash" @@ -27,13 +28,13 @@ func (n mockTraversal) SerializeForNetwork() (ipsl.AstNode, error) { return n.Serialize() } -func (n mockTraversal) Traverse(c cid.Cid, data []byte) ([]ipsl.CidTraversalPair, error) { +func (n mockTraversal) Traverse(b blocks.Block) ([]ipsl.CidTraversalPair, error) { var bad bool - if !bytes.Equal(data, n.expectedData) { + if data := b.RawData(); !bytes.Equal(data, n.expectedData) { n.t.Errorf("got wrong bytes in Traverse: expected %#v; got %#v", n.expectedData, data) bad = true } - if !c.Equals(n.expectedCid) { + if c := b.Cid(); !c.Equals(n.expectedCid) { n.t.Errorf("got wrong cid: expected %v; got %v", n.expectedCid, c) bad = true } @@ -46,12 +47,27 @@ func (n mockTraversal) Traverse(c cid.Cid, data []byte) ([]ipsl.CidTraversalPair type mockByteBlockGetter map[cid.Cid][]byte -func (g mockByteBlockGetter) GetBlock(_ context.Context, c cid.Cid) ([]byte, error) { +func (g mockByteBlockGetter) GetBlock(_ context.Context, c cid.Cid) (blocks.Block, error) { b, ok := g[c] if !ok { panic(fmt.Sprintf("missing block requested %v", c)) } - return b, nil + return blocks.NewBlockWithCid(b, c) +} + +func (g mockByteBlockGetter) GetBlocks(ctx context.Context, cids []cid.Cid) <-chan blocks.Block { + r := make(chan blocks.Block, len(cids)) + defer close(r) + + for _, c := range cids { + b, err := g.GetBlock(ctx, c) + if err != nil { + continue + } + r <- b + } + + return r } func TestSyncDFS(t *testing.T) { @@ -119,8 +135,9 @@ func TestSyncDFS(t *testing.T) { }} var result []cid.Cid - err = SyncDFS(ctx, root1Cid, traversal, getter, 10, func(c cid.Cid, data []byte) error { - if realBytes := getter[c]; !bytes.Equal(data, realBytes) { + err = SyncDFS(ctx, root1Cid, traversal, getter, 10, func(b blocks.Block) error { + c := b.Cid() + if realBytes, data := getter[c], b.RawData(); !bytes.Equal(data, realBytes) { t.Errorf("got wrong bytes in callBack: expected %#v; got %#v", realBytes, data) } diff --git a/ipsl/ipsl.go b/ipsl/ipsl.go index b58d6827e..7695130a0 100644 --- a/ipsl/ipsl.go +++ b/ipsl/ipsl.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" ) type CidTraversalPair struct { @@ -16,7 +17,7 @@ type Traversal interface { // Traverse must never be called with bytes not matching the cid. // The bytes must never be modified by the implementations. - Traverse(cid.Cid, []byte) ([]CidTraversalPair, error) + Traverse(blocks.Block) ([]CidTraversalPair, error) } type Node interface { @@ -61,7 +62,8 @@ func CompileAll(scopeName string, arguments ...SomeNode) (SomeNode, error) { return SomeNode{All(traversals...)}, nil } -func (n AllNode) Traverse(c cid.Cid, _ []byte) ([]CidTraversalPair, error) { +func (n AllNode) Traverse(b blocks.Block) ([]CidTraversalPair, error) { + c := b.Cid() r := make([]CidTraversalPair, len(n.Traversals)) for i, t := range n.Traversals { r[i] = CidTraversalPair{c, t} @@ -137,7 +139,7 @@ func Empty() Traversal { return EmptyTraversal{} } -func (c EmptyTraversal) Traverse(_ cid.Cid, _ []byte) ([]CidTraversalPair, error) { +func (c EmptyTraversal) Traverse(_ blocks.Block) ([]CidTraversalPair, error) { return []CidTraversalPair{}, nil } diff --git a/ipsl/ipsl_test.go b/ipsl/ipsl_test.go index ea24492d7..bd3092eb6 100644 --- a/ipsl/ipsl_test.go +++ b/ipsl/ipsl_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" . "github.com/ipfs/go-libipfs/ipsl" ) @@ -156,7 +157,7 @@ func TestEmpty(t *testing.T) { } } - cids, err := trav.Traverse(cid.MustParse("bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"), []byte{}) + cids, err := trav.Traverse(blocks.NewBlock([]byte("some bytes"))) if err != nil { t.Fatalf("unexpected error: %s", err.Error()) } diff --git a/ipsl/unixfs/unixfs.go b/ipsl/unixfs/unixfs.go index 75ad3b448..6b9bb594a 100644 --- a/ipsl/unixfs/unixfs.go +++ b/ipsl/unixfs/unixfs.go @@ -4,11 +4,11 @@ package unixfs import ( "fmt" - "github.com/ipfs/go-libipfs/ipsl" - unixfs_pb "github.com/ipfs/go-unixfs/pb" - "github.com/gogo/protobuf/proto" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/ipsl" + unixfs_pb "github.com/ipfs/go-unixfs/pb" "github.com/ipfs/go-merkledag/pb" ) @@ -35,13 +35,13 @@ func (n EverythingNode) SerializeForNetwork() (ipsl.AstNode, error) { return n.Serialize() } -func (n EverythingNode) Traverse(c cid.Cid, bytes []byte) ([]ipsl.CidTraversalPair, error) { - switch codec := c.Prefix().Codec; codec { +func (n EverythingNode) Traverse(b blocks.Block) ([]ipsl.CidTraversalPair, error) { + switch codec := b.Cid().Prefix().Codec; codec { case cid.Raw: return []ipsl.CidTraversalPair{}, nil case cid.DagProtobuf: var dagpb merkledag_pb.PBNode - err := proto.Unmarshal(bytes, &dagpb) + err := proto.Unmarshal(b.RawData(), &dagpb) if err != nil { return nil, fmt.Errorf("error parsing dagpb node: %w", err) } diff --git a/ipsl/unixfs/unixfs_test.go b/ipsl/unixfs/unixfs_test.go index cce5059a4..be2a306ce 100644 --- a/ipsl/unixfs/unixfs_test.go +++ b/ipsl/unixfs/unixfs_test.go @@ -6,19 +6,19 @@ import ( "os" "testing" - "github.com/ipfs/go-libipfs/ipsl/helpers" - . "github.com/ipfs/go-libipfs/ipsl/unixfs" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-exchange-offline" + "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/ipsl/helpers" + . "github.com/ipfs/go-libipfs/ipsl/unixfs" "github.com/ipld/go-car/v2" "golang.org/x/exp/slices" ) -func getSmallTreeDatastore(t *testing.T) (helpers.ByteBlockGetter, []cid.Cid) { +func getSmallTreeDatastore(t *testing.T) (blockservice.BlockGetter, []cid.Cid) { f, err := os.Open("testdata/small-tree.car") if err != nil { t.Fatalf("to open small-tree.car: %s", err) @@ -52,16 +52,16 @@ BlockLoop: cids = append(cids, block.Cid()) } - service := blockservice.New(bs, offline.Exchange(blockstore.NewBlockstore(datastore.NewNullDatastore()))) - return helpers.BlockGetterToByteBlockGetter{BlockGetter: service}, cids + return blockservice.New(bs, offline.Exchange(blockstore.NewBlockstore(datastore.NewNullDatastore()))), cids } func TestEverything(t *testing.T) { bs, expectedOrder := getSmallTreeDatastore(t) root := expectedOrder[0] var result []cid.Cid - err := helpers.SyncDFS(context.Background(), root, Everything(), bs, 10, func(c cid.Cid, data []byte) error { - hashedData, err := c.Prefix().Sum(data) + err := helpers.SyncDFS(context.Background(), root, Everything(), bs, 10, func(b blocks.Block) error { + c := b.Cid() + hashedData, err := c.Prefix().Sum(b.RawData()) if err != nil { t.Errorf("error hashing data in callBack: %s", err) } else {