Merge pull request #2962 from ipfs/kevina/block-rm

Add "ipfs block rm" command.

blocks/blockstore/arc_cache.go

@@ -32,7 +32,7 @@ func (b *arccache) DeleteBlock(k key.Key) error {
 	switch err {
 	case nil, ds.ErrNotFound, ErrNotFound:
 		b.arc.Add(k, false)
-		return nil
+		return err
 	default:
 		return err
 	}

core/commands/block.go

@@ -9,8 +9,11 @@ import (
 	"strings"
 
 	"github.com/ipfs/go-ipfs/blocks"
+	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
 	key "github.com/ipfs/go-ipfs/blocks/key"
 	cmds "github.com/ipfs/go-ipfs/commands"
+	"github.com/ipfs/go-ipfs/pin"
+	ds "gx/ipfs/QmTxLSvdhwg68WJimdS6icLPhZi28aTp6b7uihC2Yb47Xk/go-datastore"
 	mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
 	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
 )
@@ -38,6 +41,7 @@ multihash.
 		"stat": blockStatCmd,
 		"get":  blockGetCmd,
 		"put":  blockPutCmd,
+		"rm":   blockRmCmd,
 	},
 }
 
@@ -185,3 +189,135 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) {
 	log.Debugf("ipfs block: got block with key: %q", b.Key())
 	return b, nil
 }
+
+var blockRmCmd = &cmds.Command{
+	Helptext: cmds.HelpText{
+		Tagline: "Remove IPFS block(s).",
+		ShortDescription: `
+'ipfs block rm' is a plumbing command for removing raw ipfs blocks.
+It takes a list of base58 encoded multihashs to remove.
+`,
+	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("hash", true, true, "Bash58 encoded multihash of block(s) to remove."),
+	},
+	Options: []cmds.Option{
+		cmds.BoolOption("force", "f", "Ignore nonexistent blocks.").Default(false),
+		cmds.BoolOption("quiet", "q", "Write minimal output.").Default(false),
+	},
+	Run: func(req cmds.Request, res cmds.Response) {
+		n, err := req.InvocContext().GetNode()
+		if err != nil {
+			res.SetError(err, cmds.ErrNormal)
+			return
+		}
+		hashes := req.Arguments()
+		force, _, _ := req.Option("force").Bool()
+		quiet, _, _ := req.Option("quiet").Bool()
+		keys := make([]key.Key, 0, len(hashes))
+		for _, hash := range hashes {
+			k := key.B58KeyDecode(hash)
+			keys = append(keys, k)
+		}
+		outChan := make(chan interface{})
+		res.SetOutput((<-chan interface{})(outChan))
+		go func() {
+			defer close(outChan)
+			pinning := n.Pinning
+			err := rmBlocks(n.Blockstore, pinning, outChan, keys, rmBlocksOpts{
+				quiet: quiet,
+				force: force,
+			})
+			if err != nil {
+				outChan <- &RemovedBlock{Error: err.Error()}
+			}
+		}()
+		return
+	},
+	PostRun: func(req cmds.Request, res cmds.Response) {
+		if res.Error() != nil {
+			return
+		}
+		outChan, ok := res.Output().(<-chan interface{})
+		if !ok {
+			res.SetError(u.ErrCast(), cmds.ErrNormal)
+			return
+		}
+		res.SetOutput(nil)
+
+		someFailed := false
+		for out := range outChan {
+			o := out.(*RemovedBlock)
+			if o.Hash == "" && o.Error != "" {
+				res.SetError(fmt.Errorf("aborted: %s", o.Error), cmds.ErrNormal)
+				return
+			} else if o.Error != "" {
+				someFailed = true
+				fmt.Fprintf(res.Stderr(), "cannot remove %s: %s\n", o.Hash, o.Error)
+			} else {
+				fmt.Fprintf(res.Stdout(), "removed %s\n", o.Hash)
+			}
+		}
+		if someFailed {
+			res.SetError(fmt.Errorf("some blocks not removed"), cmds.ErrNormal)
+		}
+	},
+	Type: RemovedBlock{},
+}
+
+type RemovedBlock struct {
+	Hash  string `json:",omitempty"`
+	Error string `json:",omitempty"`
+}
+
+type rmBlocksOpts struct {
+	quiet bool
+	force bool	
+}
+
+func rmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, keys []key.Key, opts rmBlocksOpts) error {
+	unlocker := blocks.GCLock()
+	defer unlocker.Unlock()
+
+	stillOkay, err := checkIfPinned(pins, keys, out)
+	if err != nil {
+		return fmt.Errorf("pin check failed: %s", err)
+	}
+
+	for _, k := range stillOkay {
+		err := blocks.DeleteBlock(k)
+		if err != nil && opts.force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
+			// ignore non-existent blocks
+		} else if err != nil {
+			out <- &RemovedBlock{Hash: k.String(), Error: err.Error()}
+		} else if !opts.quiet {
+			out <- &RemovedBlock{Hash: k.String()}
+		}
+	}
+	return nil
+}
+
+func checkIfPinned(pins pin.Pinner, keys []key.Key, out chan<- interface{}) ([]key.Key, error) {
+	stillOkay := make([]key.Key, 0, len(keys))
+	res, err := pins.CheckIfPinned(keys...)
+	if err != nil {
+		return nil, err
+	}
+	for _, r := range res {
+		switch r.Mode {
+		case pin.NotPinned:
+			stillOkay = append(stillOkay, r.Key)
+		case pin.Indirect:
+			out <- &RemovedBlock{
+				Hash:  r.Key.String(),
+				Error: fmt.Sprintf("pinned via %s", r.Via)}
+		default:
+			modeStr, _ := pin.PinModeToString(r.Mode)
+			out <- &RemovedBlock{
+				Hash:  r.Key.String(),
+				Error: fmt.Sprintf("pinned: %s", modeStr)}
+
+		}
+	}
+	return stillOkay, nil
+}

pin/pin.go

@@ -75,6 +75,10 @@ type Pinner interface {
 	Pin(context.Context, *mdag.Node, bool) error
 	Unpin(context.Context, key.Key, bool) error
 
+	// Check if a set of keys are pinned, more efficient than
+	// calling IsPinned for each key
+	CheckIfPinned(keys ...key.Key) ([]Pinned, error)
+
 	// PinWithMode is for manually editing the pin structure. Use with
 	// care! If used improperly, garbage collection may not be
 	// successful.
@@ -90,6 +94,12 @@ type Pinner interface {
 	InternalPins() []key.Key
 }
 
+type Pinned struct {
+	Key  key.Key
+	Mode PinMode
+	Via  key.Key
+}
+
 // pinner implements the Pinner interface
 type pinner struct {
 	lock       sync.RWMutex
@@ -255,6 +265,70 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error)
 	return "", false, nil
 }
 
+func (p *pinner) CheckIfPinned(keys ...key.Key) ([]Pinned, error) {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+	pinned := make([]Pinned, 0, len(keys))
+	toCheck := make(map[key.Key]struct{})
+
+	// First check for non-Indirect pins directly
+	for _, k := range keys {
+		if p.recursePin.HasKey(k) {
+			pinned = append(pinned, Pinned{Key: k, Mode: Recursive})
+		} else if p.directPin.HasKey(k) {
+			pinned = append(pinned, Pinned{Key: k, Mode: Direct})
+		} else if p.isInternalPin(k) {
+			pinned = append(pinned, Pinned{Key: k, Mode: Internal})
+		} else {
+			toCheck[k] = struct{}{}
+		}
+	}
+
+	// Now walk all recursive pins to check for indirect pins
+	var checkChildren func(key.Key, key.Key) error
+	checkChildren = func(rk key.Key, parentKey key.Key) error {
+		parent, err := p.dserv.Get(context.Background(), parentKey)
+		if err != nil {
+			return err
+		}
+		for _, lnk := range parent.Links {
+			k := key.Key(lnk.Hash)
+
+			if _, found := toCheck[k]; found {
+				pinned = append(pinned,
+					Pinned{Key: k, Mode: Indirect, Via: rk})
+				delete(toCheck, k)
+			}
+
+			err := checkChildren(rk, k)
+			if err != nil {
+				return err
+			}
+
+			if len(toCheck) == 0 {
+				return nil
+			}
+		}
+		return nil
+	}
+	for _, rk := range p.recursePin.GetKeys() {
+		err := checkChildren(rk, rk)
+		if err != nil {
+			return nil, err
+		}
+		if len(toCheck) == 0 {
+			break
+		}
+	}
+
+	// Anything left in toCheck is not pinned
+	for k, _ := range toCheck {
+		pinned = append(pinned, Pinned{Key: k, Mode: NotPinned})
+	}
+
+	return pinned, nil
+}
+
 func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
 	p.lock.Lock()
 	defer p.lock.Unlock()