From eb7da51203ea9bbb6fe0cce8997c9e96d2b99821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Sep 2017 14:30:06 +0200 Subject: [PATCH 1/3] dag: Support multiple files in put MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/dag/dag.go | 107 ++++++++++++++++++++++++------------- test/sharness/t0053-dag.sh | 17 +++++- 2 files changed, 84 insertions(+), 40 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index c2b2b23581a..1d2653831e9 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -8,12 +8,15 @@ import ( "strings" cmds "github.com/ipfs/go-ipfs/commands" + files "github.com/ipfs/go-ipfs/commands/files" coredag "github.com/ipfs/go-ipfs/core/coredag" path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" + u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" + "reflect" ) var DagCmd = &cmds.Command{ @@ -53,7 +56,7 @@ into an object of the specified format. `, }, Arguments: []cmds.Argument{ - cmds.FileArg("object data", true, false, "The object to put").EnableStdin(), + cmds.FileArg("object data", true, true, "The object to put").EnableStdin(), }, Options: []cmds.Option{ cmds.StringOption("format", "f", "Format that the object will be added as.").Default("cbor"), @@ -68,12 +71,6 @@ into an object of the specified format. return } - fi, err := req.Files().NextFile() - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - ienc, _, _ := req.Option("input-enc").String() format, _, _ := req.Option("format").String() hash, _, err := req.Option("hash").String() @@ -100,52 +97,86 @@ into an object of the specified format. defer n.Blockstore.PinLock().Unlock() } - nds, err := coredag.ParseInputs(ienc, format, fi, mhType, -1) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return - } - if len(nds) == 0 { - res.SetError(fmt.Errorf("no node returned from ParseInputs"), cmds.ErrNormal) - return - } + outChan := make(chan interface{}, 8) + res.SetOutput((<-chan interface{})(outChan)) - b := n.DAG.Batch() - for _, nd := range nds { - _, err := b.Add(nd) - if err != nil { - res.SetError(err, cmds.ErrNormal) - return + addAllAndPin := func(f files.File) error { + for { + file, err := f.NextFile() + if err == io.EOF { + // Finished the list of files. + break + } else if err != nil { + return err + } + + nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1) + if err != nil { + return err + } + if len(nds) == 0 { + return fmt.Errorf("no node returned from ParseInputs") + } + + b := n.DAG.Batch() + for _, nd := range nds { + _, err := b.Add(nd) + if err != nil { + return err + } + } + + if err := b.Commit(); err != nil { + return err + } + + root := nds[0].Cid() + if dopin { + n.Pinning.PinWithMode(root, pin.Recursive) + + err := n.Pinning.Flush() + if err != nil { + return err + } + } + + outChan <- &OutputObject{Cid: root} } - } - if err := b.Commit(); err != nil { - res.SetError(err, cmds.ErrNormal) - return + return nil } - root := nds[0].Cid() - if dopin { - n.Pinning.PinWithMode(root, pin.Recursive) - - err := n.Pinning.Flush() - if err != nil { + go func() { + defer close(outChan) + if err := addAllAndPin(req.Files()); err != nil { res.SetError(err, cmds.ErrNormal) return } - } - - res.SetOutput(&OutputObject{Cid: root}) + }() }, Type: OutputObject{}, Marshalers: cmds.MarshalerMap{ cmds.Text: func(res cmds.Response) (io.Reader, error) { - oobj, ok := res.Output().(*OutputObject) + outChan, ok := res.Output().(<-chan interface{}) if !ok { - return nil, fmt.Errorf("expected a different object in marshaler") + fmt.Println(reflect.TypeOf(res.Output())) + return nil, u.ErrCast() + } + + marshal := func(v interface{}) (io.Reader, error) { + obj, ok := v.(*OutputObject) + if !ok { + return nil, u.ErrCast() + } + + return strings.NewReader(obj.Cid.String() + "\n"), nil } - return strings.NewReader(oobj.Cid.String()), nil + return &cmds.ChannelMarshaler{ + Channel: outChan, + Marshaler: marshal, + Res: res, + }, nil }, }, } diff --git a/test/sharness/t0053-dag.sh b/test/sharness/t0053-dag.sh index 5df5cc41832..1e909342708 100755 --- a/test/sharness/t0053-dag.sh +++ b/test/sharness/t0053-dag.sh @@ -153,7 +153,7 @@ test_dag_cmd() { ' test_expect_success "dag put with dag-pb works output looks good" ' - printf $HASH > dag_put_exp && + echo $HASH > dag_put_exp && test_cmp dag_put_exp dag_put_out ' @@ -163,7 +163,20 @@ test_dag_cmd() { ' test_expect_success "dag put with dag-pb works output looks good" ' - printf $HASH > dag_put_exp && + echo $HASH > dag_put_exp && + test_cmp dag_put_exp dag_put_out + ' + + test_expect_success "dag put multiple files" ' + printf {\"foo\":\"bar\"} > a.json && + printf {\"foo\":\"baz\"} > b.json && + ipfs dag put a.json b.json > dag_put_out + ' + + test_expect_success "dag put multiple files output looks good" ' + echo zdpuAoKMEvka7gKGSjF9B3of1F5gE5MyMMywxTC13wCmouQrf > dag_put_exp && + echo zdpuAogmDEvpvGjMFsNTGDEU1JMYe6v69oxR8nG81EurmGHMj >> dag_put_exp && + test_cmp dag_put_exp dag_put_out ' From 03cf65bfb98dcf6a06197553686365c2d0552be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Sep 2017 20:45:14 +0200 Subject: [PATCH 2/3] dag: fix pin lock usage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/dag/dag.go | 48 +++++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 1d2653831e9..a6e57530da9 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -16,7 +16,6 @@ import ( cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid" u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util" mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash" - "reflect" ) var DagCmd = &cmds.Command{ @@ -93,10 +92,6 @@ into an object of the specified format. } } - if dopin { - defer n.Blockstore.PinLock().Unlock() - } - outChan := make(chan interface{}, 8) res.SetOutput((<-chan interface{})(outChan)) @@ -118,29 +113,37 @@ into an object of the specified format. return fmt.Errorf("no node returned from ParseInputs") } - b := n.DAG.Batch() - for _, nd := range nds { - _, err := b.Add(nd) - if err != nil { - return err + var cid *cid.Cid + err = func() error { + if dopin { + defer n.Blockstore.PinLock().Unlock() } - } - - if err := b.Commit(); err != nil { - return err - } - root := nds[0].Cid() - if dopin { - n.Pinning.PinWithMode(root, pin.Recursive) + b := n.DAG.Batch() + for _, nd := range nds { + _, err := b.Add(nd) + if err != nil { + return err + } + } - err := n.Pinning.Flush() - if err != nil { + if err := b.Commit(); err != nil { return err } - } - outChan <- &OutputObject{Cid: root} + cid = nds[0].Cid() + if dopin { + n.Pinning.PinWithMode(cid, pin.Recursive) + + err := n.Pinning.Flush() + if err != nil { + return err + } + } + return nil + }() + + outChan <- &OutputObject{Cid: cid} } return nil @@ -159,7 +162,6 @@ into an object of the specified format. cmds.Text: func(res cmds.Response) (io.Reader, error) { outChan, ok := res.Output().(<-chan interface{}) if !ok { - fmt.Println(reflect.TypeOf(res.Output())) return nil, u.ErrCast() } From f5cbc4c896c5f48b7cd9a3555471e3adcd390371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 21 Sep 2017 23:22:50 +0200 Subject: [PATCH 3/3] dag: batch multiple files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/dag/dag.go | 48 ++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index a6e57530da9..e7f8b9176a7 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -96,6 +96,9 @@ into an object of the specified format. res.SetOutput((<-chan interface{})(outChan)) addAllAndPin := func(f files.File) error { + cids := cid.NewSet() + b := n.DAG.Batch() + for { file, err := f.NextFile() if err == io.EOF { @@ -113,37 +116,34 @@ into an object of the specified format. return fmt.Errorf("no node returned from ParseInputs") } - var cid *cid.Cid - err = func() error { - if dopin { - defer n.Blockstore.PinLock().Unlock() + for _, nd := range nds { + _, err := b.Add(nd) + if err != nil { + return err } + } - b := n.DAG.Batch() - for _, nd := range nds { - _, err := b.Add(nd) - if err != nil { - return err - } - } + cid := nds[0].Cid() + cids.Add(cid) + outChan <- &OutputObject{Cid: cid} + } - if err := b.Commit(); err != nil { - return err - } + if err := b.Commit(); err != nil { + return err + } - cid = nds[0].Cid() - if dopin { - n.Pinning.PinWithMode(cid, pin.Recursive) + if dopin { + defer n.Blockstore.PinLock().Unlock() - err := n.Pinning.Flush() - if err != nil { - return err - } - } + cids.ForEach(func(c *cid.Cid) error { + n.Pinning.PinWithMode(c, pin.Recursive) return nil - }() + }) - outChan <- &OutputObject{Cid: cid} + err := n.Pinning.Flush() + if err != nil { + return err + } } return nil