Merge pull request #5834 from ipfs/feat/urlstore-pin

add pinning support to the urlstore
ipfs · Dec 12, 2018 · cb57105 · cb57105
2 parents e18328b + edd37d8
commit cb57105
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 9 deletions.
diff --git a/core/commands/urlstore.go b/core/commands/urlstore.go
@@ -7,6 +7,7 @@ import (
 
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
 	filestore "github.com/ipfs/go-ipfs/filestore"
+	pin "github.com/ipfs/go-ipfs/pin"
 
 	chunk "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker"
 	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
@@ -36,16 +37,14 @@ control.
 The file is added using raw-leaves but otherwise using the default
 settings for 'ipfs add'.
 
-The file is not pinned, so this command should be followed by an 'ipfs
-pin add'.
-
 This command is considered temporary until a better solution can be
 found.  It may disappear or the semantics can change at any
 time.
 `,
 	},
 	Options: []cmdkit.Option{
 		cmdkit.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),
+		cmdkit.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
 	},
 	Arguments: []cmdkit.Argument{
 		cmdkit.StringArg("url", true, false, "URL to add to IPFS"),
@@ -73,6 +72,7 @@ time.
 		}
 
 		useTrickledag, _ := req.Options[trickleOptionName].(bool)
+		dopin, _ := req.Options[pinOptionName].(bool)
 
 		hreq, err := http.NewRequest("GET", url, nil)
 		if err != nil {
@@ -87,6 +87,11 @@ time.
 			return fmt.Errorf("expected code 200, got: %d", hres.StatusCode)
 		}
 
+		if dopin {
+			// Take the pinlock
+			defer n.Blockstore.PinLock().Unlock()
+		}
+
 		chk := chunk.NewSizeSplitter(hres.Body, chunk.DefaultBlockSize)
 		prefix := cid.NewPrefixV1(cid.DagProtobuf, mh.SHA2_256)
 		dbp := &ihelper.DagBuilderParams{
@@ -102,13 +107,22 @@ time.
 		if useTrickledag {
 			layout = trickle.Layout
 		}
+
 		root, err := layout(dbp.New(chk))
 		if err != nil {
 			return err
 		}
 
+		c := root.Cid()
+		if dopin {
+			n.Pinning.PinWithMode(c, pin.Recursive)
+			if err := n.Pinning.Flush(); err != nil {
+				return err
+			}
+		}
+
 		return cmds.EmitOnce(res, &BlockStat{
-			Key:  root.Cid().String(),
+			Key:  c.String(),
 			Size: int(hres.ContentLength),
 		})
 	},

diff --git a/test/sharness/t0272-urlstore.sh b/test/sharness/t0272-urlstore.sh
@@ -46,9 +46,8 @@ test_expect_success "enable urlstore" '
 test_launch_ipfs_daemon --offline
 
 test_expect_success "add files using gateway address via url store" '
-  HASH1=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) &&
-  HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a) &&
-  ipfs pin add $HASH1 $HASH2
+  HASH1=$(ipfs urlstore add --pin=false http://127.0.0.1:$GWAY_PORT/ipfs/$HASH1a) &&
+  HASH2=$(ipfs urlstore add http://127.0.0.1:$GWAY_PORT/ipfs/$HASH2a)
 '
 
 test_expect_success "make sure hashes are different" '
@@ -86,6 +85,21 @@ test_expect_success "ipfs filestore verify works with urls" '
   test_cmp verify_expect verify_actual
 '
 
+test_expect_success "garbage collect file1 from the urlstore" '
+  ipfs repo gc > /dev/null
+'
+
+test_expect_success "can no longer retrieve file1 from urlstore" '
+  rm -f file1.actual &&
+  test_must_fail ipfs get $HASH1 -o file1.actual
+'
+
+test_expect_success "can still retrieve file2 from urlstore" '
+  rm -f file2.actual &&
+  ipfs get $HASH2 -o file2.actual &&
+  test_cmp file2 file2.actual
+'
+
 test_expect_success "remove original hashes from local gateway" '
   ipfs pin rm $HASH1a $HASH2a &&
   ipfs repo gc > /dev/null
@@ -99,7 +113,6 @@ test_expect_success "gatway no longer has files" '
 cat <<EOF | sort > verify_expect_2
 error   zb2rhX1q5oFFzEkPNsTe1Y8osUdFqSQGjUWRZsqC9fbY6WVSk  262144 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 0
 error   zb2rhYbKFn1UWGHXaAitcdVTkDGTykX8RFpGWzRFuLpoe9VE4  237856 http://127.0.0.1:$GWAY_PORT/ipfs/QmUow2T4P69nEsqTQDZCt8yg9CPS8GFmpuDAr5YtsPhTdM 262144
-error   zb2rhjddJ5DNzBrFu8G6CP1ApY25BukwCeskXHzN1H18CiVVZ    2222 http://127.0.0.1:$GWAY_PORT/ipfs/QmcHm3BL2cXuQ6rJdKQgPrmT9suqGkfy2KzH3MkXPEBXU6 0
 EOF
 
 test_expect_success "ipfs filestore verify is correct" '
@@ -113,7 +126,7 @@ test_expect_success "files can not be retrieved via the urlstore" '
 '
 
 test_expect_success "remove broken files" '
-  ipfs pin rm $HASH1 $HASH2 &&
+  ipfs pin rm $HASH2 &&
   ipfs repo gc > /dev/null
 '