From e2cd36e88f9e9587404c874ce36e9ffe5578d103 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Jun 2017 18:02:23 -0700 Subject: [PATCH 1/2] pinning + pathresolver: fix pinning/unpinning of sharded directories * Change ResolveToCid to take a Resolver and a NameSystem instead of an ipfs Node. * Make the pin/unpin methods use a Unixfs path resolver. Closes: #3974 License: MIT Signed-off-by: Steven Allen --- core/commands/pin.go | 17 ++++++++++++++--- core/corerepo/pinning.go | 18 +++++++++++++++--- core/pathresolver.go | 14 +++++++------- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/core/commands/pin.go b/core/commands/pin.go index 8f64863463e..272ac772f0f 100644 --- a/core/commands/pin.go +++ b/core/commands/pin.go @@ -12,6 +12,7 @@ import ( dag "github.com/ipfs/go-ipfs/merkledag" path "github.com/ipfs/go-ipfs/path" pin "github.com/ipfs/go-ipfs/pin" + uio "github.com/ipfs/go-ipfs/unixfs/io" context "context" u "gx/ipfs/QmWbjfz3u6HkAdPh34dgPchGbQjob6LXLhAeCGii2TX69n/go-ipfs-util" @@ -377,13 +378,18 @@ new pin and removing the old one. return } - fromc, err := core.ResolveToCid(req.Context(), n, from) + r := &path.Resolver{ + DAG: n.DAG, + ResolveOnce: uio.ResolveUnixfsOnce, + } + + fromc, err := core.ResolveToCid(req.Context(), n.Namesys, r, from) if err != nil { res.SetError(err, cmds.ErrNormal) return } - toc, err := core.ResolveToCid(req.Context(), n, to) + toc, err := core.ResolveToCid(req.Context(), n.Namesys, r, to) if err != nil { res.SetError(err, cmds.ErrNormal) return @@ -486,13 +492,18 @@ func pinLsKeys(args []string, typeStr string, ctx context.Context, n *core.IpfsN keys := make(map[string]RefKeyObject) + r := &path.Resolver{ + DAG: n.DAG, + ResolveOnce: uio.ResolveUnixfsOnce, + } + for _, p := range args { pth, err := path.ParsePath(p) if err != nil { return nil, err } - c, err := core.ResolveToCid(ctx, n, pth) + c, err := core.ResolveToCid(ctx, n.Namesys, r, pth) if err != nil { return nil, err } diff --git a/core/corerepo/pinning.go b/core/corerepo/pinning.go index 72993eb79db..908e7efaf3c 100644 --- a/core/corerepo/pinning.go +++ b/core/corerepo/pinning.go @@ -19,6 +19,7 @@ import ( "github.com/ipfs/go-ipfs/core" path "github.com/ipfs/go-ipfs/path" + uio "github.com/ipfs/go-ipfs/unixfs/io" cid "gx/ipfs/QmYhQaCYEcaPPjxJX7YcPcVKkQfRy6sJ7B3XmGFk82XYdQ/go-cid" node "gx/ipfs/Qmb3Hm9QDFmfYuET4pu7Kyg8JV78jFa1nvZx5vnCZsK4ck/go-ipld-format" @@ -26,13 +27,19 @@ import ( func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) { dagnodes := make([]node.Node, 0) + + r := &path.Resolver{ + DAG: n.DAG, + ResolveOnce: uio.ResolveUnixfsOnce, + } + for _, fpath := range paths { p, err := path.ParsePath(fpath) if err != nil { return nil, err } - dagnode, err := core.Resolve(ctx, n.Namesys, n.Resolver, p) + dagnode, err := core.Resolve(ctx, n.Namesys, r, p) if err != nil { return nil, fmt.Errorf("pin: %s", err) } @@ -61,15 +68,20 @@ func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) } func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) { - var unpinned []*cid.Cid + + r := &path.Resolver{ + DAG: n.DAG, + ResolveOnce: uio.ResolveUnixfsOnce, + } + for _, p := range paths { p, err := path.ParsePath(p) if err != nil { return nil, err } - k, err := core.ResolveToCid(ctx, n, p) + k, err := core.ResolveToCid(ctx, n.Namesys, r, p) if err != nil { return nil, err } diff --git a/core/pathresolver.go b/core/pathresolver.go index d5336bc2db8..830150b537c 100644 --- a/core/pathresolver.go +++ b/core/pathresolver.go @@ -57,14 +57,14 @@ func Resolve(ctx context.Context, nsys namesys.NameSystem, r *path.Resolver, p p return r.ResolvePath(ctx, p) } -// ResolveToKey resolves a path to a key. +// ResolveToCid resolves a path to a cid. // -// It first checks if the path is already in the form of just a key ( or -// /ipfs/) and returns immediately if so. Otherwise, it falls back onto +// It first checks if the path is already in the form of just a cid ( or +// /ipfs/) and returns immediately if so. Otherwise, it falls back onto // Resolve to perform resolution of the dagnode being referenced. -func ResolveToCid(ctx context.Context, n *IpfsNode, p path.Path) (*cid.Cid, error) { +func ResolveToCid(ctx context.Context, nsys namesys.NameSystem, r *path.Resolver, p path.Path) (*cid.Cid, error) { - // If the path is simply a key, parse and return it. Parsed paths are already + // If the path is simply a cid, parse and return it. Parsed paths are already // normalized (read: prepended with /ipfs/ if needed), so segment[1] should // always be the key. if p.IsJustAKey() { @@ -77,12 +77,12 @@ func ResolveToCid(ctx context.Context, n *IpfsNode, p path.Path) (*cid.Cid, erro if err != nil { return nil, err } - dagnode, err := Resolve(ctx, n.Namesys, n.Resolver, head) + dagnode, err := Resolve(ctx, nsys, r, head) if err != nil { return nil, err } - // Extract and return the key of the link to the target dag node. + // Extract and return the cid of the link to the target dag node. link, _, err := dagnode.ResolveLink([]string{tail}) if err != nil { return nil, err From 8c78d4072dcf57fba697a8342711005011826bad Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 12 Jun 2017 19:02:22 -0700 Subject: [PATCH 2/2] pinning: test pinning/unpinning files in sharded directories License: MIT Signed-off-by: Steven Allen --- test/sharness/t0250-files-api.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/sharness/t0250-files-api.sh b/test/sharness/t0250-files-api.sh index dd48de2ffbe..ffd5bc8adc6 100755 --- a/test/sharness/t0250-files-api.sh +++ b/test/sharness/t0250-files-api.sh @@ -72,6 +72,16 @@ test_sharding() { test_cmp file_out file_exp ' + test_expect_success "can pin a file from sharded directory" ' + ipfs files stat --hash /foo/file42 > pin_file_hash && + ipfs pin add < pin_file_hash > pin_hash + ' + + test_expect_success "can unpin a file from sharded directory" ' + read -r _ HASH _ < pin_hash && + ipfs pin rm $HASH + ' + test_expect_success "output object was really sharded" ' ipfs files stat --hash /foo > expected_foo_hash && echo QmPkwLJTYZRGPJ8Lazr9qPdrLmswPtUjaDbEpmR9jEh1se > actual_foo_hash &&