From ee6472904e4d00d4992c2434369d37735caa5fbd Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 15 Feb 2018 00:38:02 +0100 Subject: [PATCH 1/5] Feat: remove circular dependencies in merkledag package tests This avoids using unixfs package and importer packages in merkledag, which removes circular depedencies making it hard to extract this module. License: MIT Signed-off-by: Hector Sanjuan --- merkledag/merkledag_test.go | 92 ++++++++++++++++++++++++++----------- 1 file changed, 64 insertions(+), 28 deletions(-) diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index db8d49bcc8b..51083859f62 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -16,14 +16,11 @@ import ( bserv "github.com/ipfs/go-ipfs/blockservice" bstest "github.com/ipfs/go-ipfs/blockservice/test" offline "github.com/ipfs/go-ipfs/exchange/offline" - imp "github.com/ipfs/go-ipfs/importer" . "github.com/ipfs/go-ipfs/merkledag" mdpb "github.com/ipfs/go-ipfs/merkledag/pb" dstest "github.com/ipfs/go-ipfs/merkledag/test" - uio "github.com/ipfs/go-ipfs/unixfs/io" u "gx/ipfs/QmNiJuT8Ja3hMVpBHXv3Q6dwmperaQ6JjLtpMQgMCD7xvx/go-ipfs-util" - chunker "gx/ipfs/QmWo8jYc19ppG7YoTsrr2kEtLRbARTJho5oNXFTR6B7Peq/go-ipfs-chunker" cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" blocks "gx/ipfs/Qmej7nf81hi2x2tvjRBF3mcp74sQyuDH4VMYDGd1YtXjb2/go-block-format" @@ -129,6 +126,61 @@ func TestBatchFetchDupBlock(t *testing.T) { runBatchFetchTest(t, read) } +// makeTestDAG creates a simple DAG from the data in a reader. +// First, a node is created from each 512 bytes of data from the reader +// (like a the Size chunker would do). Then all nodes are added as children +// to a root node, which is returned. +func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { + p := make([]byte, 512) + nodes := []*ProtoNode{} + var err error = nil + _, err = read.Read(p) + for err == nil { + protoNode := NodeWithData(p) + nodes = append(nodes, protoNode) + _, err = read.Read(p) + } + + if err != io.EOF { + t.Fatal(err) + } + + ctx := context.Background() + // Add a root referencing all created nodes + root := NodeWithData(nil) + for _, n := range nodes { + root.AddNodeLink(n.Cid().String(), n) + err := ds.Add(ctx, n) + if err != nil { + t.Fatal(err) + } + } + err = ds.Add(ctx, root) + if err != nil { + t.Fatal(err) + } + return root +} + +// makeTestDAGReader takes the root node as returned by makeTestDAG and +// provides a reader that reads all the RawData from that node and its children. +func makeTestDAGReader(t *testing.T, root ipld.Node, ds ipld.DAGService) io.Reader { + ctx := context.Background() + buf := new(bytes.Buffer) + buf.Write(root.RawData()) + for _, l := range root.Links() { + n, err := ds.Get(ctx, l.Cid) + if err != nil { + t.Fatal(err) + } + _, err = buf.Write(n.RawData()) + if err != nil { + t.Fatal(err) + } + } + return buf +} + func runBatchFetchTest(t *testing.T, read io.Reader) { ctx := context.Background() var dagservs []ipld.DAGService @@ -136,19 +188,11 @@ func runBatchFetchTest(t *testing.T, read io.Reader) { dagservs = append(dagservs, NewDAGService(bsi)) } - spl := chunker.NewSizeSplitter(read, 512) - - root, err := imp.BuildDagFromReader(dagservs[0], spl) - if err != nil { - t.Fatal(err) - } + root := makeTestDAG(t, read, dagservs[0]) t.Log("finished setup.") - dagr, err := uio.NewDagReader(ctx, root, dagservs[0]) - if err != nil { - t.Fatal(err) - } + dagr := makeTestDAGReader(t, root, dagservs[0]) expected, err := ioutil.ReadAll(dagr) if err != nil { @@ -181,11 +225,9 @@ func runBatchFetchTest(t *testing.T, read io.Reader) { if !ok { errs <- ErrNotProtobuf } - - read, err := uio.NewDagReader(ctx, firstpb, dagservs[i]) - if err != nil { - errs <- err - } + _ = firstpb + _ = expected + read := makeTestDAGReader(t, firstpb, dagservs[i]) datagot, err := ioutil.ReadAll(read) if err != nil { errs <- err @@ -228,12 +270,9 @@ func TestFetchGraph(t *testing.T) { } read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) - root, err := imp.BuildDagFromReader(dservs[0], chunker.NewSizeSplitter(read, 512)) - if err != nil { - t.Fatal(err) - } + root := makeTestDAG(t, read, dservs[0]) - err = FetchGraph(context.TODO(), root.Cid(), dservs[1]) + err := FetchGraph(context.TODO(), root.Cid(), dservs[1]) if err != nil { t.Fatal(err) } @@ -254,14 +293,11 @@ func TestEnumerateChildren(t *testing.T) { ds := NewDAGService(bsi[0]) read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024) - root, err := imp.BuildDagFromReader(ds, chunker.NewSizeSplitter(read, 512)) - if err != nil { - t.Fatal(err) - } + root := makeTestDAG(t, read, ds) set := cid.NewSet() - err = EnumerateChildren(context.Background(), ds.GetLinks, root.Cid(), set.Visit) + err := EnumerateChildren(context.Background(), ds.GetLinks, root.Cid(), set.Visit) if err != nil { t.Fatal(err) } From c42e684e6be1042cc17345b215e6cb3a7feacc2a Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 15 Feb 2018 20:08:37 +0100 Subject: [PATCH 2/5] Golint merkledag_test.go License: MIT Signed-off-by: Hector Sanjuan --- merkledag/merkledag_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 51083859f62..c80fa80b7e1 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -133,7 +133,7 @@ func TestBatchFetchDupBlock(t *testing.T) { func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { p := make([]byte, 512) nodes := []*ProtoNode{} - var err error = nil + var err error _, err = read.Read(p) for err == nil { protoNode := NodeWithData(p) From 701fb82f9318da00b22a98cbef039e3642df657c Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 27 Feb 2018 18:37:50 +0100 Subject: [PATCH 3/5] merkledag_test: address #4704 review comments License: MIT Signed-off-by: Hector Sanjuan --- merkledag/merkledag_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index c80fa80b7e1..ec7465162ef 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -134,7 +134,7 @@ func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { p := make([]byte, 512) nodes := []*ProtoNode{} var err error - _, err = read.Read(p) + _, err = io.ReadFull(read, p) for err == nil { protoNode := NodeWithData(p) nodes = append(nodes, protoNode) @@ -225,8 +225,6 @@ func runBatchFetchTest(t *testing.T, read io.Reader) { if !ok { errs <- ErrNotProtobuf } - _ = firstpb - _ = expected read := makeTestDAGReader(t, firstpb, dagservs[i]) datagot, err := ioutil.ReadAll(read) if err != nil { From 050a699bc8a13b769cc742b63db15a0789d2a6b0 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Tue, 27 Feb 2018 18:40:02 +0100 Subject: [PATCH 4/5] merkledag_test.go: Handle short reads in makeTestDAG License: MIT Signed-off-by: Hector Sanjuan --- merkledag/merkledag_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index ec7465162ef..cae3566fc81 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -134,7 +134,10 @@ func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { p := make([]byte, 512) nodes := []*ProtoNode{} var err error - _, err = io.ReadFull(read, p) + n, err = io.ReadFull(read, p) + if n != len(p) { + t.Fatal("should have read 512 bytes from the reader") + } for err == nil { protoNode := NodeWithData(p) nodes = append(nodes, protoNode) From cd9c8c5d45a949761bcbdd0c9c00df35bc120ddc Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 1 Mar 2018 14:17:22 +0100 Subject: [PATCH 5/5] Use ReadFull. Remove duplicated code License: MIT Signed-off-by: Hector Sanjuan --- merkledag/merkledag_test.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index cae3566fc81..d67a20aa7b9 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -133,19 +133,23 @@ func TestBatchFetchDupBlock(t *testing.T) { func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { p := make([]byte, 512) nodes := []*ProtoNode{} - var err error - n, err = io.ReadFull(read, p) - if n != len(p) { - t.Fatal("should have read 512 bytes from the reader") - } - for err == nil { + + for { + n, err := io.ReadFull(read, p) + if err == io.EOF { + break + } + + if err != nil { + t.Fatal(err) + } + + if n != len(p) { + t.Fatal("should have read 512 bytes from the reader") + } + protoNode := NodeWithData(p) nodes = append(nodes, protoNode) - _, err = read.Read(p) - } - - if err != io.EOF { - t.Fatal(err) } ctx := context.Background() @@ -158,7 +162,7 @@ func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { t.Fatal(err) } } - err = ds.Add(ctx, root) + err := ds.Add(ctx, root) if err != nil { t.Fatal(err) }