From bf23516dc0dea968ebe9458b7a1ad3ccc39258d1 Mon Sep 17 00:00:00 2001 From: Jeromy Date: Sun, 10 Jul 2016 00:40:23 -0700 Subject: [PATCH] cache encoded data when reading dag nodes from disk License: MIT Signed-off-by: Jeromy --- core/commands/add.go | 3 ++- core/commands/files/files.go | 4 ++-- core/commands/ls.go | 2 +- core/commands/object/object.go | 13 +++++------ core/commands/object/patch.go | 8 +++---- core/commands/unixfs/ls.go | 4 ++-- core/corehttp/gateway_handler.go | 2 +- core/coreunix/add.go | 11 ++------- core/coreunix/metadata.go | 4 ++-- fuse/ipns/common.go | 3 +-- fuse/ipns/ipns_unix.go | 2 +- fuse/readonly/readonly_unix.go | 2 +- importer/helpers/helpers.go | 4 ++-- importer/trickle/trickle_test.go | 4 ++-- importer/trickle/trickledag.go | 4 ++-- merkledag/coding.go | 11 ++++++--- merkledag/merkledag.go | 4 ++++ merkledag/merkledag_test.go | 18 +++++++-------- merkledag/node.go | 26 ++++++++++++++++----- merkledag/traverse/traverse_test.go | 14 +++++------ merkledag/utils/utils_test.go | 6 ++--- mfs/dir.go | 5 ++-- mfs/file.go | 4 ++-- mfs/mfs_test.go | 36 +++++++++++++++-------------- mfs/system.go | 2 +- namesys/publisher.go | 2 +- path/resolver_test.go | 4 ++-- pin/pin_test.go | 4 ++-- pin/set.go | 18 +++++++-------- tar/format.go | 8 +++---- unixfs/archive/tar/writer.go | 2 +- unixfs/format.go | 5 ++++ unixfs/io/dagreader.go | 4 ++-- unixfs/io/dirbuilder.go | 4 +++- unixfs/mod/dagmodifier.go | 15 ++++++------ unixfs/mod/dagmodifier_test.go | 4 ++-- 36 files changed, 144 insertions(+), 122 deletions(-) diff --git a/core/commands/add.go b/core/commands/add.go index 5daf7170510..d0c7596fc46 100644 --- a/core/commands/add.go +++ b/core/commands/add.go @@ -12,6 +12,7 @@ import ( core "github.com/ipfs/go-ipfs/core" dagtest "github.com/ipfs/go-ipfs/merkledag/test" mfs "github.com/ipfs/go-ipfs/mfs" + ft "github.com/ipfs/go-ipfs/unixfs" u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) @@ -160,7 +161,7 @@ You can now refer to the added file in a gateway, like so: if hash { md := dagtest.Mock() - mr, err := mfs.NewRoot(req.Context(), md, coreunix.NewDirNode(), nil) + mr, err := mfs.NewRoot(req.Context(), md, ft.EmptyDirNode(), nil) if err != nil { res.SetError(err, cmds.ErrNormal) return diff --git a/core/commands/files/files.go b/core/commands/files/files.go index 4f2fcab083d..f6f94a3ea43 100644 --- a/core/commands/files/files.go +++ b/core/commands/files/files.go @@ -163,7 +163,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) { return nil, err } - d, err := ft.FromBytes(nd.Data) + d, err := ft.FromBytes(nd.Data()) if err != nil { return nil, err } @@ -871,7 +871,7 @@ func getFileHandle(r *mfs.Root, path string, create bool) (*mfs.File, error) { return nil, fmt.Errorf("%s was not a directory", dirname) } - nd := &dag.Node{Data: ft.FilePBData(nil, 0)} + nd := dag.NodeWithData(ft.FilePBData(nil, 0)) err = pdir.AddChild(fname, nd) if err != nil { return nil, err diff --git a/core/commands/ls.go b/core/commands/ls.go index b9ccf1e73f6..e304e678bbc 100644 --- a/core/commands/ls.go +++ b/core/commands/ls.go @@ -110,7 +110,7 @@ format: } } if linkNode != nil { - d, err := unixfs.FromBytes(linkNode.Data) + d, err := unixfs.FromBytes(linkNode.Data()) if err != nil { res.SetError(err, cmds.ErrNormal) return diff --git a/core/commands/object/object.go b/core/commands/object/object.go index c08d15d09a9..46fc389cb8e 100644 --- a/core/commands/object/object.go +++ b/core/commands/object/object.go @@ -93,7 +93,7 @@ is the raw data of the object. res.SetError(err, cmds.ErrNormal) return } - res.SetOutput(bytes.NewReader(node.Data)) + res.SetOutput(bytes.NewReader(node.Data())) }, } @@ -198,7 +198,7 @@ This command outputs data in the following encodings: node := &Node{ Links: make([]Link, len(object.Links)), - Data: string(object.Data), + Data: string(object.Data()), } for i, link := range object.Links { @@ -438,9 +438,7 @@ Available templates: func nodeFromTemplate(template string) (*dag.Node, error) { switch template { case "unixfs-dir": - nd := new(dag.Node) - nd.Data = ft.FolderPBData() - return nd, nil + return ft.EmptyDirNode(), nil default: return nil, fmt.Errorf("template '%s' not found", template) } @@ -566,9 +564,10 @@ func deserializeNode(node *Node, dataFieldEncoding string) (*dag.Node, error) { dagnode := new(dag.Node) switch dataFieldEncoding { case "text": - dagnode.Data = []byte(node.Data) + dagnode.SetData([]byte(node.Data)) case "base64": - dagnode.Data, _ = base64.StdEncoding.DecodeString(node.Data) + data, _ := base64.StdEncoding.DecodeString(node.Data) + dagnode.SetData(data) default: return nil, fmt.Errorf("Unkown data field encoding") } diff --git a/core/commands/object/patch.go b/core/commands/object/patch.go index ce87a048c8b..623186a119a 100644 --- a/core/commands/object/patch.go +++ b/core/commands/object/patch.go @@ -91,7 +91,7 @@ the limit will not be respected by the network. return } - rootnd.Data = append(rootnd.Data, data...) + rootnd.SetData(append(rootnd.Data(), data...)) newkey, err := nd.DAG.Add(rootnd) if err != nil { @@ -153,7 +153,7 @@ Example: return } - root.Data = data + root.SetData(data) newkey, err := nd.DAG.Add(root) if err != nil { @@ -287,9 +287,7 @@ to a file containing 'bar', and returns the hash of the new object. var createfunc func() *dag.Node if create { - createfunc = func() *dag.Node { - return &dag.Node{Data: ft.FolderPBData()} - } + createfunc = ft.EmptyDirNode } e := dagutils.NewDagEditor(root, nd.DAG) diff --git a/core/commands/unixfs/ls.go b/core/commands/unixfs/ls.go index d9e72a87917..ff3e9594de7 100644 --- a/core/commands/unixfs/ls.go +++ b/core/commands/unixfs/ls.go @@ -101,7 +101,7 @@ Example: continue } - unixFSNode, err := unixfs.FromBytes(merkleNode.Data) + unixFSNode, err := unixfs.FromBytes(merkleNode.Data()) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -128,7 +128,7 @@ Example: res.SetError(err, cmds.ErrNormal) return } - d, err := unixfs.FromBytes(linkNode.Data) + d, err := unixfs.FromBytes(linkNode.Data()) if err != nil { res.SetError(err, cmds.ErrNormal) return diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 8313e877e25..88b7a85877d 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -391,7 +391,7 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) { case nil: // object set-data case - rnode.Data = newnode.Data + rnode.SetData(newnode.Data()) newkey, err = i.node.DAG.Add(rnode) if err != nil { diff --git a/core/coreunix/add.go b/core/coreunix/add.go index d0c68685a4e..8ad900c9648 100644 --- a/core/coreunix/add.go +++ b/core/coreunix/add.go @@ -29,8 +29,6 @@ import ( var log = logging.Logger("coreunix") -var folderData = unixfs.FolderPBData() - // how many bytes of progress to wait before sending a progress update message const progressReaderIncrement = 1024 * 256 @@ -67,7 +65,7 @@ type AddedObject struct { } func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) { - mr, err := mfs.NewRoot(ctx, ds, NewDirNode(), nil) + mr, err := mfs.NewRoot(ctx, ds, unixfs.EmptyDirNode(), nil) if err != nil { return nil, err } @@ -392,7 +390,7 @@ func (adder *Adder) addFile(file files.File) error { return err } - dagnode := &dag.Node{Data: sdata} + dagnode := dag.NodeWithData(sdata) _, err = adder.dagService.Add(dagnode) if err != nil { return err @@ -488,11 +486,6 @@ func NewMemoryDagService() dag.DAGService { return dag.NewDAGService(bsrv) } -// TODO: generalize this to more than unix-fs nodes. -func NewDirNode() *dag.Node { - return &dag.Node{Data: unixfs.FolderPBData()} -} - // from core/commands/object.go func getOutput(dagnode *dag.Node) (*Object, error) { key, err := dagnode.Key() diff --git a/core/coreunix/metadata.go b/core/coreunix/metadata.go index e3f392aadc2..eb318e1a2eb 100644 --- a/core/coreunix/metadata.go +++ b/core/coreunix/metadata.go @@ -21,7 +21,7 @@ func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error return "", err } - mdnode.Data = mdata + mdnode.SetData(mdata) if err := mdnode.AddNodeLinkClean("file", nd); err != nil { return "", err } @@ -42,5 +42,5 @@ func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) { return nil, err } - return ft.MetadataFromBytes(nd.Data) + return ft.MetadataFromBytes(nd.Data()) } diff --git a/fuse/ipns/common.go b/fuse/ipns/common.go index 1791bf1e227..fc06865ebc5 100644 --- a/fuse/ipns/common.go +++ b/fuse/ipns/common.go @@ -4,7 +4,6 @@ import ( context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context" "github.com/ipfs/go-ipfs/core" - mdag "github.com/ipfs/go-ipfs/merkledag" nsys "github.com/ipfs/go-ipfs/namesys" path "github.com/ipfs/go-ipfs/path" ft "github.com/ipfs/go-ipfs/unixfs" @@ -14,7 +13,7 @@ import ( // InitializeKeyspace sets the ipns record for the given key to // point to an empty directory. func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error { - emptyDir := &mdag.Node{Data: ft.FolderPBData()} + emptyDir := ft.EmptyDirNode() nodek, err := n.DAG.Add(emptyDir) if err != nil { return err diff --git a/fuse/ipns/ipns_unix.go b/fuse/ipns/ipns_unix.go index e1423046470..992730ce35c 100644 --- a/fuse/ipns/ipns_unix.go +++ b/fuse/ipns/ipns_unix.go @@ -481,7 +481,7 @@ func (fi *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { func (dir *Directory) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { // New 'empty' file - nd := &dag.Node{Data: ft.FilePBData(nil, 0)} + nd := dag.NodeWithData(ft.FilePBData(nil, 0)) err := dir.dir.AddChild(req.Name, nd) if err != nil { return nil, nil, err diff --git a/fuse/readonly/readonly_unix.go b/fuse/readonly/readonly_unix.go index 314d5ff3486..31f43b0fe59 100644 --- a/fuse/readonly/readonly_unix.go +++ b/fuse/readonly/readonly_unix.go @@ -84,7 +84,7 @@ type Node struct { func (s *Node) loadData() error { s.cached = new(ftpb.Data) - return proto.Unmarshal(s.Nd.Data, s.cached) + return proto.Unmarshal(s.Nd.Data(), s.cached) } // Attr returns the attributes of a given node. diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go index 29983795c5f..9d66b55f751 100644 --- a/importer/helpers/helpers.go +++ b/importer/helpers/helpers.go @@ -59,7 +59,7 @@ func NewUnixfsBlock() *UnixfsNode { // NewUnixfsNodeFromDag reconstructs a Unixfs node from a given dag node func NewUnixfsNodeFromDag(nd *dag.Node) (*UnixfsNode, error) { - mb, err := ft.FSNodeFromBytes(nd.Data) + mb, err := ft.FSNodeFromBytes(nd.Data()) if err != nil { return nil, err } @@ -126,6 +126,6 @@ func (n *UnixfsNode) GetDagNode() (*dag.Node, error) { if err != nil { return nil, err } - n.node.Data = data + n.node.SetData(data) return n.node, nil } diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index 1dd4350882c..bdddeaf59e2 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -524,7 +524,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { data := []byte("AB") nd := new(merkledag.Node) - nd.Data = ft.FilePBData(nil, 0) + nd.SetData(ft.FilePBData(nil, 0)) dbp := &h.DagBuilderParams{ Dagserv: ds, @@ -562,7 +562,7 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { } func printDag(nd *merkledag.Node, ds merkledag.DAGService, indent int) { - pbd, err := ft.FromBytes(nd.Data) + pbd, err := ft.FromBytes(nd.Data()) if err != nil { panic(err) } diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go index 8955568da10..f22715fd642 100644 --- a/importer/trickle/trickledag.go +++ b/importer/trickle/trickledag.go @@ -241,7 +241,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi return errors.New("expected direct block") } - pbn, err := ft.FromBytes(nd.Data) + pbn, err := ft.FromBytes(nd.Data()) if err != nil { return err } @@ -253,7 +253,7 @@ func verifyTDagRec(nd *dag.Node, depth, direct, layerRepeat int, ds dag.DAGServi } // Verify this is a branch node - pbn, err := ft.FromBytes(nd.Data) + pbn, err := ft.FromBytes(nd.Data()) if err != nil { return err } diff --git a/merkledag/coding.go b/merkledag/coding.go index 10c30727aa2..2c92b559f3a 100644 --- a/merkledag/coding.go +++ b/merkledag/coding.go @@ -33,7 +33,8 @@ func (n *Node) unmarshal(encoded []byte) error { } sort.Stable(LinkSlice(n.Links)) // keep links sorted - n.Data = pbn.GetData() + n.data = pbn.GetData() + n.encoded = encoded return nil } @@ -62,8 +63,8 @@ func (n *Node) getPBNode() *pb.PBNode { pbn.Links[i].Hash = []byte(l.Hash) } - if len(n.Data) > 0 { - pbn.Data = n.Data + if len(n.data) > 0 { + pbn.Data = n.data } return pbn } @@ -73,11 +74,15 @@ func (n *Node) getPBNode() *pb.PBNode { func (n *Node) EncodeProtobuf(force bool) ([]byte, error) { sort.Stable(LinkSlice(n.Links)) // keep links sorted if n.encoded == nil || force { + n.cached = nil var err error n.encoded, err = n.Marshal() if err != nil { return nil, err } + } + + if n.cached == nil { n.cached = u.Hash(n.encoded) } diff --git a/merkledag/merkledag.go b/merkledag/merkledag.go index 470f45faaab..835d26cbf6b 100644 --- a/merkledag/merkledag.go +++ b/merkledag/merkledag.go @@ -93,6 +93,9 @@ func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) { } return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err) } + + res.cached = k.ToMultihash() + return res, nil } @@ -147,6 +150,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO out <- &NodeOption{Err: err} return } + nd.cached = b.Key().ToMultihash() // buffered, no need to select out <- &NodeOption{Node: nd} diff --git a/merkledag/merkledag_test.go b/merkledag/merkledag_test.go index 72bc3ef1700..8a679335af1 100644 --- a/merkledag/merkledag_test.go +++ b/merkledag/merkledag_test.go @@ -46,9 +46,9 @@ func getDagservAndPinner(t *testing.T) dagservAndPinner { func TestNode(t *testing.T) { - n1 := &Node{Data: []byte("beep")} - n2 := &Node{Data: []byte("boop")} - n3 := &Node{Data: []byte("beep boop")} + n1 := NodeWithData([]byte("beep")) + n2 := NodeWithData([]byte("boop")) + n3 := NodeWithData([]byte("beep boop")) if err := n3.AddNodeLink("beep-link", n1); err != nil { t.Error(err) } @@ -58,7 +58,7 @@ func TestNode(t *testing.T) { printn := func(name string, n *Node) { fmt.Println(">", name) - fmt.Println("data:", string(n.Data)) + fmt.Println("data:", string(n.Data())) fmt.Println("links:") for _, l := range n.Links { @@ -118,8 +118,8 @@ func SubtestNodeStat(t *testing.T, n *Node) { expected := NodeStat{ NumLinks: len(n.Links), BlockSize: len(enc), - LinksSize: len(enc) - len(n.Data), // includes framing. - DataSize: len(n.Data), + LinksSize: len(enc) - len(n.Data()), // includes framing. + DataSize: len(n.Data()), CumulativeSize: int(cumSize), Hash: k.B58String(), } @@ -255,7 +255,7 @@ func TestEmptyKey(t *testing.T) { func TestCantGet(t *testing.T) { dsp := getDagservAndPinner(t) - a := &Node{Data: []byte("A")} + a := NodeWithData([]byte("A")) k, err := a.Key() if err != nil { @@ -339,7 +339,7 @@ func TestFetchFailure(t *testing.T) { top := new(Node) for i := 0; i < 10; i++ { - nd := &Node{Data: []byte{byte('a' + i)}} + nd := NodeWithData([]byte{byte('a' + i)}) _, err := ds.Add(nd) if err != nil { t.Fatal(err) @@ -352,7 +352,7 @@ func TestFetchFailure(t *testing.T) { } for i := 0; i < 10; i++ { - nd := &Node{Data: []byte{'f', 'a' + byte(i)}} + nd := NodeWithData([]byte{'f', 'a' + byte(i)}) _, err := ds_bad.Add(nd) if err != nil { t.Fatal(err) diff --git a/merkledag/node.go b/merkledag/node.go index 36479fb752e..7be5c4d0a83 100644 --- a/merkledag/node.go +++ b/merkledag/node.go @@ -15,7 +15,7 @@ var ErrLinkNotFound = fmt.Errorf("no link by that name") // nodes have opaque data and a set of navigable links. type Node struct { Links []*Link - Data []byte + data []byte // cache encoded/marshaled value encoded []byte @@ -78,6 +78,10 @@ func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) { return serv.Get(ctx, key.Key(l.Hash)) } +func NodeWithData(d []byte) *Node { + return &Node{data: d} +} + // AddNodeLink adds a link to another node. func (n *Node) AddNodeLink(name string, that *Node) error { n.encoded = nil @@ -168,9 +172,9 @@ func (n *Node) GetLinkedNode(ctx context.Context, ds DAGService, name string) (* // NOTE: Does not make copies of Node objects in the links. func (n *Node) Copy() *Node { nnode := new(Node) - if len(n.Data) > 0 { - nnode.Data = make([]byte, len(n.Data)) - copy(nnode.Data, n.Data) + if len(n.data) > 0 { + nnode.data = make([]byte, len(n.data)) + copy(nnode.data, n.data) } if len(n.Links) > 0 { @@ -180,6 +184,16 @@ func (n *Node) Copy() *Node { return nnode } +func (n *Node) Data() []byte { + return n.data +} + +func (n *Node) SetData(d []byte) { + n.encoded = nil + n.cached = nil + n.data = d +} + // UpdateNodeLink return a copy of the node with the link name set to point to // that. If a link of the same name existed, it is removed. func (n *Node) UpdateNodeLink(name string, that *Node) (*Node, error) { @@ -226,8 +240,8 @@ func (n *Node) Stat() (*NodeStat, error) { Hash: key.B58String(), NumLinks: len(n.Links), BlockSize: len(enc), - LinksSize: len(enc) - len(n.Data), // includes framing. - DataSize: len(n.Data), + LinksSize: len(enc) - len(n.data), // includes framing. + DataSize: len(n.data), CumulativeSize: int(cumSize), }, nil } diff --git a/merkledag/traverse/traverse_test.go b/merkledag/traverse/traverse_test.go index 5ca906a5188..2bd3444116e 100644 --- a/merkledag/traverse/traverse_test.go +++ b/merkledag/traverse/traverse_test.go @@ -326,7 +326,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte) buf := new(bytes.Buffer) walk := func(current State) error { - s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data) + s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data()) t.Logf("walk: %s", s) buf.Write([]byte(s)) return nil @@ -349,7 +349,7 @@ func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte) } func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node { - a := &mdag.Node{Data: []byte("/a")} + a := mdag.NodeWithData([]byte("/a")) addLink(t, ds, a, child(t, ds, a, "aa")) addLink(t, ds, a, child(t, ds, a, "ab")) addLink(t, ds, a, child(t, ds, a, "ac")) @@ -358,7 +358,7 @@ func newFan(t *testing.T, ds mdag.DAGService) *mdag.Node { } func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node { - a := &mdag.Node{Data: []byte("/a")} + a := mdag.NodeWithData([]byte("/a")) aa := child(t, ds, a, "aa") aaa := child(t, ds, aa, "aaa") aaaa := child(t, ds, aaa, "aaaa") @@ -371,7 +371,7 @@ func newLinkedList(t *testing.T, ds mdag.DAGService) *mdag.Node { } func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node { - a := &mdag.Node{Data: []byte("/a")} + a := mdag.NodeWithData([]byte("/a")) aa := child(t, ds, a, "aa") ab := child(t, ds, a, "ab") addLink(t, ds, aa, child(t, ds, aa, "aaa")) @@ -384,7 +384,7 @@ func newBinaryTree(t *testing.T, ds mdag.DAGService) *mdag.Node { } func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node { - a := &mdag.Node{Data: []byte("/a")} + a := mdag.NodeWithData([]byte("/a")) aa := child(t, ds, a, "aa") aaa := child(t, ds, aa, "aaa") aaaa := child(t, ds, aaa, "aaaa") @@ -401,7 +401,7 @@ func newBinaryDAG(t *testing.T, ds mdag.DAGService) *mdag.Node { } func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) { - to := string(a.Data) + "2" + string(b.Data) + to := string(a.Data()) + "2" + string(b.Data()) if _, err := ds.Add(b); err != nil { t.Error(err) } @@ -411,5 +411,5 @@ func addLink(t *testing.T, ds mdag.DAGService, a, b *mdag.Node) { } func child(t *testing.T, ds mdag.DAGService, a *mdag.Node, name string) *mdag.Node { - return &mdag.Node{Data: []byte(string(a.Data) + "/" + name)} + return mdag.NodeWithData([]byte(string(a.Data()) + "/" + name)) } diff --git a/merkledag/utils/utils_test.go b/merkledag/utils/utils_test.go index b225a3dfff3..1ec444b0ba1 100644 --- a/merkledag/utils/utils_test.go +++ b/merkledag/utils/utils_test.go @@ -13,9 +13,7 @@ import ( func TestAddLink(t *testing.T) { ds := mdtest.Mock() - fishnode := &dag.Node{ - Data: []byte("fishcakes!"), - } + fishnode := dag.NodeWithData([]byte("fishcakes!")) fk, err := ds.Add(fishnode) if err != nil { @@ -90,7 +88,7 @@ func TestInsertNode(t *testing.T) { } func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) { - child := &dag.Node{Data: []byte(data)} + child := dag.NodeWithData([]byte(data)) ck, err := e.tmp.Add(child) if err != nil { t.Fatal(err) diff --git a/mfs/dir.go b/mfs/dir.go index fba61ea4c49..cf178e4527e 100644 --- a/mfs/dir.go +++ b/mfs/dir.go @@ -121,7 +121,7 @@ func (d *Directory) childNode(name string) (FSNode, error) { // cacheNode caches a node into d.childDirs or d.files and returns the FSNode. func (d *Directory) cacheNode(name string, nd *dag.Node) (FSNode, error) { - i, err := ft.FromBytes(nd.Data) + i, err := ft.FromBytes(nd.Data()) if err != nil { return nil, err } @@ -268,7 +268,8 @@ func (d *Directory) Mkdir(name string) (*Directory, error) { } } - ndir := &dag.Node{Data: ft.FolderPBData()} + ndir := new(dag.Node) + ndir.SetData(ft.FolderPBData()) _, err = d.dserv.Add(ndir) if err != nil { diff --git a/mfs/file.go b/mfs/file.go index 216bdfa7516..46ca7314b0f 100644 --- a/mfs/file.go +++ b/mfs/file.go @@ -45,7 +45,7 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { node := fi.node fi.nodelk.Unlock() - fsn, err := ft.FSNodeFromBytes(node.Data) + fsn, err := ft.FSNodeFromBytes(node.Data()) if err != nil { return nil, err } @@ -86,7 +86,7 @@ func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) { func (fi *File) Size() (int64, error) { fi.nodelk.Lock() defer fi.nodelk.Unlock() - pbd, err := ft.FromBytes(fi.node.Data) + pbd, err := ft.FromBytes(fi.node.Data()) if err != nil { return 0, err } diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 84ce2266845..3069fb13c30 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -31,6 +31,10 @@ import ( u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util" ) +func emptyDirNode() *dag.Node { + return dag.NodeWithData(ft.FolderPBData()) +} + func getDagserv(t *testing.T) dag.DAGService { db := dssync.MutexWrap(ds.NewMapDatastore()) bs := bstore.NewBlockstore(db) @@ -183,7 +187,7 @@ func catNode(ds dag.DAGService, nd *dag.Node) ([]byte, error) { func setupRoot(ctx context.Context, t *testing.T) (dag.DAGService, *Root) { ds := getDagserv(t) - root := &dag.Node{Data: ft.FolderPBData()} + root := emptyDirNode() rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, k key.Key) error { fmt.Println("PUBLISHED: ", k) return nil @@ -282,7 +286,7 @@ func TestDirectoryLoadFromDag(t *testing.T) { t.Fatal(err) } - dir := &dag.Node{Data: ft.FolderPBData()} + dir := emptyDirNode() _, err = ds.Add(dir) if err != nil { t.Fatal(err) @@ -293,17 +297,15 @@ func TestDirectoryLoadFromDag(t *testing.T) { t.Fatal(err) } - top := &dag.Node{ - Data: ft.FolderPBData(), - Links: []*dag.Link{ - &dag.Link{ - Name: "a", - Hash: fihash, - }, - &dag.Link{ - Name: "b", - Hash: dirhash, - }, + top := emptyDirNode() + top.Links = []*dag.Link{ + &dag.Link{ + Name: "a", + Hash: fihash, + }, + &dag.Link{ + Name: "b", + Hash: dirhash, }, } @@ -540,7 +542,7 @@ func actorMakeFile(d *Directory) error { } name := randomName() - f, err := NewFile(name, &dag.Node{Data: ft.FilePBData(nil, 0)}, d, d.dserv) + f, err := NewFile(name, dag.NodeWithData(ft.FilePBData(nil, 0)), d, d.dserv) if err != nil { return err } @@ -756,7 +758,7 @@ func TestFlushing(t *testing.T) { e := mkdirP(t, dir, "a/b/e") data := []byte("this is a test\n") - nd1 := &dag.Node{Data: ft.FilePBData(data, uint64(len(data)))} + nd1 := dag.NodeWithData(ft.FilePBData(data, uint64(len(data)))) if err := c.AddChild("TEST", nd1); err != nil { t.Fatal(err) @@ -792,7 +794,7 @@ func TestFlushing(t *testing.T) { t.Fatal(err) } - fsnode, err := ft.FSNodeFromBytes(rnd.Data) + fsnode, err := ft.FSNodeFromBytes(rnd.Data()) if err != nil { t.Fatal(err) } @@ -897,7 +899,7 @@ func TestFileDescriptors(t *testing.T) { ds, rt := setupRoot(ctx, t) dir := rt.GetValue().(*Directory) - nd := &dag.Node{Data: ft.FilePBData(nil, 0)} + nd := dag.NodeWithData(ft.FilePBData(nil, 0)) fi, err := NewFile("test", nd, dir, ds) if err != nil { t.Fatal(err) diff --git a/mfs/system.go b/mfs/system.go index 15e7c768497..40d9d29cdb7 100644 --- a/mfs/system.go +++ b/mfs/system.go @@ -83,7 +83,7 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.Node, pf PubFu dserv: ds, } - pbn, err := ft.FromBytes(node.Data) + pbn, err := ft.FromBytes(node.Data()) if err != nil { log.Error("IPNS pointer was not unixfs node") return nil, err diff --git a/namesys/publisher.go b/namesys/publisher.go index 65c48265572..eebb3153ed3 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -330,7 +330,7 @@ func ValidateIpnsRecord(k key.Key, val []byte) error { // point to an empty directory. // TODO: this doesnt feel like it belongs here func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, pins pin.Pinner, key ci.PrivKey) error { - emptyDir := &dag.Node{Data: ft.FolderPBData()} + emptyDir := ft.EmptyDirNode() nodek, err := ds.Add(emptyDir) if err != nil { return err diff --git a/path/resolver_test.go b/path/resolver_test.go index fe8155a85a6..735a79e6d2b 100644 --- a/path/resolver_test.go +++ b/path/resolver_test.go @@ -15,8 +15,8 @@ import ( func randNode() (*merkledag.Node, key.Key) { node := new(merkledag.Node) - node.Data = make([]byte, 32) - util.NewTimeSeededRand().Read(node.Data) + node.SetData(make([]byte, 32)) + util.NewTimeSeededRand().Read(node.Data()) k, _ := node.Key() return node, k } diff --git a/pin/pin_test.go b/pin/pin_test.go index a6ec85dac6a..1430061c277 100644 --- a/pin/pin_test.go +++ b/pin/pin_test.go @@ -18,8 +18,8 @@ import ( func randNode() (*mdag.Node, key.Key) { nd := new(mdag.Node) - nd.Data = make([]byte, 32) - util.NewTimeSeededRand().Read(nd.Data) + nd.SetData(make([]byte, 32)) + util.NewTimeSeededRand().Read(nd.Data()) k, _ := nd.Key() return nd, k } diff --git a/pin/set.go b/pin/set.go index fec38e25472..7257ccaecb9 100644 --- a/pin/set.go +++ b/pin/set.go @@ -111,7 +111,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint if err := writeHdr(n, hdr); err != nil { return nil, err } - hdrLen := len(n.Data) + hdrLen := len(n.Data()) if estimatedLen < maxItems { // it'll probably fit @@ -122,12 +122,12 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint break } n.Links = append(n.Links, &merkledag.Link{Hash: k.ToMultihash()}) - n.Data = append(n.Data, data...) + n.SetData(append(n.Data(), data...)) } // sort by hash, also swap item Data s := sortByHash{ links: n.Links[defaultFanout:], - data: n.Data[hdrLen:], + data: n.Data()[hdrLen:], } sort.Stable(s) } @@ -179,11 +179,11 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint } func readHdr(n *merkledag.Node) (*pb.Set, []byte, error) { - hdrLenRaw, consumed := binary.Uvarint(n.Data) + hdrLenRaw, consumed := binary.Uvarint(n.Data()) if consumed <= 0 { return nil, nil, errors.New("invalid Set header length") } - buf := n.Data[consumed:] + buf := n.Data()[consumed:] if hdrLenRaw > uint64(len(buf)) { return nil, nil, errors.New("impossibly large Set header length") } @@ -209,10 +209,10 @@ func writeHdr(n *merkledag.Node, hdr *pb.Set) error { if err != nil { return err } - n.Data = make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData)) - written := binary.PutUvarint(n.Data, uint64(len(hdrData))) - n.Data = n.Data[:written] - n.Data = append(n.Data, hdrData...) + n.SetData(make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData))) + written := binary.PutUvarint(n.Data(), uint64(len(hdrData))) + n.SetData(n.Data()[:written]) + n.SetData(append(n.Data(), hdrData...)) return nil } diff --git a/tar/format.go b/tar/format.go index 07783733a37..a185af6d773 100644 --- a/tar/format.go +++ b/tar/format.go @@ -45,7 +45,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) { tr := tar.NewReader(r) root := new(dag.Node) - root.Data = []byte("ipfs/tar") + root.SetData([]byte("ipfs/tar")) e := dagutil.NewDagEditor(root, ds) @@ -65,7 +65,7 @@ func ImportTar(r io.Reader, ds dag.DAGService) (*dag.Node, error) { return nil, err } - header.Data = headerBytes + header.SetData(headerBytes) if h.Size > 0 { spl := chunk.NewRabin(tr, uint64(chunk.DefaultBlockSize)) @@ -170,7 +170,7 @@ func (tr *tarReader) Read(b []byte) (int, error) { return 0, err } - tr.hdrBuf = bytes.NewReader(headerNd.Data) + tr.hdrBuf = bytes.NewReader(headerNd.Data()) dataNd, err := headerNd.GetLinkedNode(tr.ctx, tr.ds, "data") if err != nil && err != dag.ErrLinkNotFound { @@ -197,7 +197,7 @@ func (tr *tarReader) Read(b []byte) (int, error) { } func ExportTar(ctx context.Context, root *dag.Node, ds dag.DAGService) (io.Reader, error) { - if string(root.Data) != "ipfs/tar" { + if string(root.Data()) != "ipfs/tar" { return nil, errors.New("not an ipfs tarchive") } return &tarReader{ diff --git a/unixfs/archive/tar/writer.go b/unixfs/archive/tar/writer.go index ad151016ad0..3d1f47eea12 100644 --- a/unixfs/archive/tar/writer.go +++ b/unixfs/archive/tar/writer.go @@ -69,7 +69,7 @@ func (w *Writer) writeFile(nd *mdag.Node, pb *upb.Data, fpath string) error { func (w *Writer) WriteNode(nd *mdag.Node, fpath string) error { pb := new(upb.Data) - if err := proto.Unmarshal(nd.Data, pb); err != nil { + if err := proto.Unmarshal(nd.Data(), pb); err != nil { return err } diff --git a/unixfs/format.go b/unixfs/format.go index f279a8843b3..0235f0c7c73 100644 --- a/unixfs/format.go +++ b/unixfs/format.go @@ -6,6 +6,7 @@ package unixfs import ( "errors" + dag "github.com/ipfs/go-ipfs/merkledag" pb "github.com/ipfs/go-ipfs/unixfs/pb" proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto" ) @@ -222,3 +223,7 @@ func BytesForMetadata(m *Metadata) ([]byte, error) { pbd.Data = mdd return proto.Marshal(pbd) } + +func EmptyDirNode() *dag.Node { + return dag.NodeWithData(FolderPBData()) +} diff --git a/unixfs/io/dagreader.go b/unixfs/io/dagreader.go index b78b46269fb..cbbe0285852 100644 --- a/unixfs/io/dagreader.go +++ b/unixfs/io/dagreader.go @@ -60,7 +60,7 @@ type ReadSeekCloser interface { // the given node, using the passed in DAGService for data retreival func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (*DagReader, error) { pb := new(ftpb.Data) - if err := proto.Unmarshal(n.Data, pb); err != nil { + if err := proto.Unmarshal(n.Data(), pb); err != nil { return nil, err } @@ -117,7 +117,7 @@ func (dr *DagReader) precalcNextBuf(ctx context.Context) error { dr.linkPosition++ pb := new(ftpb.Data) - err = proto.Unmarshal(nxt.Data, pb) + err = proto.Unmarshal(nxt.Data(), pb) if err != nil { return fmt.Errorf("incorrectly formatted protobuf: %s", err) } diff --git a/unixfs/io/dirbuilder.go b/unixfs/io/dirbuilder.go index 8dad9a16ddc..3db0b9ef968 100644 --- a/unixfs/io/dirbuilder.go +++ b/unixfs/io/dirbuilder.go @@ -15,7 +15,9 @@ type directoryBuilder struct { // NewEmptyDirectory returns an empty merkledag Node with a folder Data chunk func NewEmptyDirectory() *mdag.Node { - return &mdag.Node{Data: format.FolderPBData()} + nd := new(mdag.Node) + nd.SetData(format.FolderPBData()) + return nd } // NewDirectory returns a directoryBuilder. It needs a DAGService to add the Children diff --git a/unixfs/mod/dagmodifier.go b/unixfs/mod/dagmodifier.go index cb3cfd5897a..66ba5f24bb9 100644 --- a/unixfs/mod/dagmodifier.go +++ b/unixfs/mod/dagmodifier.go @@ -139,7 +139,7 @@ func (dm *DagModifier) Write(b []byte) (int, error) { } func (dm *DagModifier) Size() (int64, error) { - pbn, err := ft.FromBytes(dm.curNode.Data) + pbn, err := ft.FromBytes(dm.curNode.Data()) if err != nil { return 0, err } @@ -207,7 +207,7 @@ func (dm *DagModifier) Sync() error { // returns the new key of the passed in node and whether or not all the data in the reader // has been consumed. func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (key.Key, bool, error) { - f, err := ft.FromBytes(node.Data) + f, err := ft.FromBytes(node.Data()) if err != nil { return "", false, err } @@ -225,7 +225,8 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) return "", false, err } - nd := &mdag.Node{Data: b} + nd := new(mdag.Node) + nd.SetData(b) k, err := dm.dagserv.Add(nd) if err != nil { return "", false, err @@ -429,12 +430,12 @@ func (dm *DagModifier) Truncate(size int64) error { func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGService) (*mdag.Node, error) { if len(nd.Links) == 0 { // TODO: this can likely be done without marshaling and remarshaling - pbn, err := ft.FromBytes(nd.Data) + pbn, err := ft.FromBytes(nd.Data()) if err != nil { return nil, err } - nd.Data = ft.WrapData(pbn.Data[:size]) + nd.SetData(ft.WrapData(pbn.Data[:size])) return nd, nil } @@ -448,7 +449,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer return nil, err } - childsize, err := ft.DataSize(child.Data) + childsize, err := ft.DataSize(child.Data()) if err != nil { return nil, err } @@ -486,7 +487,7 @@ func dagTruncate(ctx context.Context, nd *mdag.Node, size uint64, ds mdag.DAGSer return nil, err } - nd.Data = d + nd.SetData(d) // invalidate cache and recompute serialized data _, err = nd.EncodeProtobuf(true) diff --git a/unixfs/mod/dagmodifier_test.go b/unixfs/mod/dagmodifier_test.go index dffd205bf1b..7ed30dd266c 100644 --- a/unixfs/mod/dagmodifier_test.go +++ b/unixfs/mod/dagmodifier_test.go @@ -156,7 +156,7 @@ func TestDagModifierBasic(t *testing.T) { t.Fatal(err) } - size, err := ft.DataSize(node.Data) + size, err := ft.DataSize(node.Data()) if err != nil { t.Fatal(err) } @@ -590,7 +590,7 @@ func arrComp(a, b []byte) error { } func printDag(nd *mdag.Node, ds mdag.DAGService, indent int) { - pbd, err := ft.FromBytes(nd.Data) + pbd, err := ft.FromBytes(nd.Data()) if err != nil { panic(err) }