feat(cmds/add): --to-files option as files cp

core/commands/add.go

@@ -13,6 +13,8 @@ import (
 	"github.com/cheggaaa/pb"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	files "github.com/ipfs/go-ipfs-files"
+	ipld "github.com/ipfs/go-ipld-format"
+	mfs "github.com/ipfs/go-mfs"
 	coreiface "github.com/ipfs/interface-go-ipfs-core"
 	"github.com/ipfs/interface-go-ipfs-core/options"
 	mh "github.com/multiformats/go-multihash"
@@ -45,6 +47,7 @@ const (
 	hashOptionName        = "hash"
 	inlineOptionName      = "inline"
 	inlineLimitOptionName = "inline-limit"
+	toFilesOptionName     = "to-files"
 )
 
 const adderOutChanSize = 8
@@ -79,6 +82,20 @@ You can now refer to the added file in a gateway, like so:
 
   /ipfs/QmaG4FuMqEBnQNn3C8XJ5bpW8kLs7zq2ZXgHptJHbKDDVx/example.jpg
 
+Files imported with 'ipfs add' are protected from GC (implicit '--pin=true'),
+but it is up to you to remember the returned CID to get the data back later.
+
+Passing '--to-files' creates a reference in Files API (MFS), making it easier
+to find it in the future:
+
+  > ipfs files mkdir -p /myfs/dir
+  > ipfs add example.jpg --to-files /myfs/dir/
+  > ipfs files ls /myfs/dir/
+  example.jpg
+
+See 'ipfs files --help' to learn more about using MFS
+for keeping track of added files and directories.
+
 The chunker option, '-s', specifies the chunking strategy that dictates
 how to break files into blocks. Blocks with same content can
 be deduplicated. Different chunking strategies will produce different
@@ -138,14 +155,15 @@ See 'dag export' and 'dag import' for more information.
 		cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),
 		cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),
 		cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),
-		cmds.BoolOption(pinOptionName, "Pin this object when adding.").WithDefault(true),
 		cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),
 		cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),
 		cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),
 		cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),
 		cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),
 		cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),
 		cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),
+		cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),
+		cmds.StringOption(toFilesOptionName, "Add reference to Files API (MFS) at the provided path."),
 	},
 	PreRun: func(req *cmds.Request, env cmds.Environment) error {
 		quiet, _ := req.Options[quietOptionName].(bool)
@@ -186,10 +204,11 @@ See 'dag export' and 'dag import' for more information.
 		hashFunStr, _ := req.Options[hashOptionName].(string)
 		inline, _ := req.Options[inlineOptionName].(bool)
 		inlineLimit, _ := req.Options[inlineLimitOptionName].(int)
+		toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)
 
 		hashFunCode, ok := mh.Names[strings.ToLower(hashFunStr)]
 		if !ok {
-			return fmt.Errorf("unrecognized hash function: %s", strings.ToLower(hashFunStr))
+			return fmt.Errorf("unrecognized hash function: %q", strings.ToLower(hashFunStr))
 		}
 
 		enc, err := cmdenv.GetCidEncoder(req)
@@ -235,7 +254,12 @@ See 'dag export' and 'dag import' for more information.
 
 		opts = append(opts, nil) // events option placeholder
 
+		ipfsNode, err := cmdenv.GetNode(env)
+		if err != nil {
+			return err
+		}
 		var added int
+		var fileAddedToMFS bool
 		addit := toadd.Entries()
 		for addit.Next() {
 			_, dir := addit.Node().(files.Directory)
@@ -246,7 +270,65 @@ See 'dag export' and 'dag import' for more information.
 			go func() {
 				var err error
 				defer close(events)
-				_, err = api.Unixfs().Add(req.Context, addit.Node(), opts...)
+				pathAdded, err := api.Unixfs().Add(req.Context, addit.Node(), opts...)
+				if err != nil {
+					errCh <- err
+					return
+				}
+
+				// creating MFS pointers when optional --to-files is set
+				if toFilesSet {
+					if toFilesStr == "" {
+						toFilesStr = "/"
+					}
+					toFilesDst, err := checkPath(toFilesStr)
+					if err != nil {
+						errCh <- fmt.Errorf("%s: %w", toFilesOptionName, err)
+						return
+					}
+					dstAsDir := toFilesDst[len(toFilesDst)-1] == '/'
+
+					if dstAsDir {
+						mfsNode, err := mfs.Lookup(ipfsNode.FilesRoot, toFilesDst)
+						// confirm dst exists
+						if err != nil {
+							errCh <- fmt.Errorf("%s: MFS destination directory %q does not exist: %w", toFilesOptionName, toFilesDst, err)
+							return
+						}
+						// confirm dst is a dir
+						if mfsNode.Type() != mfs.TDir {
+							errCh <- fmt.Errorf("%s: MFS destination %q is not a directory", toFilesOptionName, toFilesDst)
+							return
+						}
+						// if MFS destination is a dir, append filename to the dir path
+						toFilesDst += path.Base(addit.Name())
+					}
+
+					// error if we try to overwrite a preexisting file destination
+					if fileAddedToMFS && !dstAsDir {
+						errCh <- fmt.Errorf("%s: MFS destination is a file: only one entry can be copied to %q", toFilesOptionName, toFilesDst)
+						return
+					}
+
+					_, err = mfs.Lookup(ipfsNode.FilesRoot, path.Dir(toFilesDst))
+					if err != nil {
+						errCh <- fmt.Errorf("%s: MFS destination parent %q %q does not exist: %w", toFilesOptionName, toFilesDst, path.Dir(toFilesDst), err)
+						return
+					}
+
+					var nodeAdded ipld.Node
+					nodeAdded, err = api.Dag().Get(req.Context, pathAdded.Cid())
+					if err != nil {
+						errCh <- err
+						return
+					}
+					err = mfs.PutNode(ipfsNode.FilesRoot, toFilesDst, nodeAdded)
+					if err != nil {
+						errCh <- fmt.Errorf("%s: cannot put node in path %q: %w", toFilesOptionName, toFilesDst, err)
+						return
+					}
+					fileAddedToMFS = true
+				}
 				errCh <- err
 			}()
 

core/commands/files.go

@@ -581,7 +581,7 @@ const (
 
 var filesReadCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
-		Tagline: "Read a file in a given MFS.",
+		Tagline: "Read a file from MFS.",
 		ShortDescription: `
 Read a specified number of bytes from a file at a given offset. By default,
 it will read the entire file similar to the Unix cat.
@@ -724,11 +724,16 @@ const (
 
 var filesWriteCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
-		Tagline: "Write to a mutable file in a given filesystem.",
+		Tagline: "Append to (modify) a file in MFS.",
 		ShortDescription: `
-Write data to a file in a given filesystem. This command allows you to specify
-a beginning offset to write to. The entire length of the input will be
-written.
+A low-level MFS command that allows you to append data to a file. If you want
+to add a file without modifying an existing one, use 'ipfs add --to-files'
+instead.
+`,
+		LongDescription: `
+A low-level MFS command that allows you to append data at the end of a file, or
+specify a beginning offset within a file to write to. The entire length of the
+input will be written.
 
 If the '--create' option is specified, the file will be created if it does not
 exist. Nonexistent intermediate directories will not be created unless the
@@ -755,6 +760,22 @@ WARNING:
 Usage of the '--flush=false' option does not guarantee data durability until
 the tree has been flushed. This can be accomplished by running 'ipfs files
 stat' on the file or any of its ancestors.
+
+WARNING:
+
+The CID produced by 'files write' will be different from 'ipfs add' because
+'ipfs file write' creates a trickle-dag optimized for append-only operations
+See '--trickle' in 'ipfs add --help' for more information.
+
+If you want to add a file without modifying an existing one,
+use 'ipfs add' with '--to-files':
+
+  > ipfs files mkdir -p /myfs/dir
+  > ipfs add example.jpg --to-files /myfs/dir/
+  > ipfs files ls /myfs/dir/
+  example.jpg
+
+See '--to-files' in 'ipfs add --help' for more information.
 `,
 	},
 	Arguments: []cmds.Argument{
@@ -1019,7 +1040,7 @@ func updatePath(rt *mfs.Root, pth string, builder cid.Builder) error {
 
 var filesRmCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
-		Tagline: "Remove a file.",
+		Tagline: "Remove a file from MFS.",
 		ShortDescription: `
 Remove files or directories.
 

test/sharness/t0040-add-and-cat.sh

@@ -362,6 +362,113 @@ test_add_cat_file() {
       rm mountdir/same-file/hello.txt  &&
       rmdir mountdir/same-file
     '
+
+  ## --to-files with single source
+
+  test_expect_success "ipfs add --to-files /mfspath succeeds" '
+    mkdir -p mountdir && echo "Hello MFS!" > mountdir/mfs.txt &&
+    ipfs add mountdir/mfs.txt --to-files /ipfs-add-to-files >actual
+  '
+
+  test_expect_success "ipfs add --to-files output looks good" '
+    HASH_MFS="QmVT8bL3sGBA2TwvX8JPhrv5CYZL8LLLfW7mxkUjPZsgBr" &&
+    echo "added $HASH_MFS mfs.txt" >expected &&
+    test_cmp expected actual
+  '
+
+  test_expect_success "ipfs files read succeeds" '
+    ipfs files read /ipfs-add-to-files >actual &&
+    ipfs files rm /ipfs-add-to-files
+  '
+
+  test_expect_success "ipfs cat output looks good" '
+    echo "Hello MFS!" >expected &&
+    test_cmp expected actual
+  '
+
+  test_expect_success "ipfs add --to-files requires argument" '
+    test_expect_code 1 ipfs add mountdir/mfs.txt --to-files >actual 2>&1 &&
+    test_should_contain "Error: missing argument for option \"to-files\"" actual
+  '
+
+  test_expect_success "ipfs add --to-files / (MFS root) works" '
+    echo "Hello MFS!" >expected &&
+    ipfs add mountdir/mfs.txt --to-files / &&
+    ipfs files read /mfs.txt >actual &&
+    test_cmp expected actual &&
+    ipfs files rm /mfs.txt &&
+    rm mountdir/mfs.txt
+  '
+
+  ## --to-files with multiple sources
+
+  test_expect_success "ipfs add file1 file2 --to-files /mfspath0 (without trailing slash) fails" '
+    mkdir -p test &&
+    echo "file1" > test/mfs1.txt &&
+    echo "file2" > test/mfs2.txt &&
+    test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfspath0 >actual 2>&1 &&
+    test_should_contain "MFS destination is a file: only one entry can be copied to \"/mfspath0\"" actual &&
+    ipfs files rm -r --force /mfspath0
+  '
+
+  test_expect_success "ipfs add file1 file2 --to-files /mfsfile1 (without trailing slash + with preexisting file) fails" '
+    echo test | ipfs files write --create /mfsfile1 &&
+    test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsfile1 >actual 2>&1 &&
+    test_should_contain "Error: to-files: cannot put node in path \"/mfsfile1\"" actual &&
+    ipfs files rm -r --force /mfsfile1
+  '
+
+  test_expect_success "ipfs add file1 file2 --to-files /mfsdir1 (without trailing slash + with preexisting dir) fails" '
+    ipfs files mkdir -p /mfsdir1 &&
+    test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsdir1 >actual 2>&1 &&
+    test_should_contain "Error: to-files: cannot put node in path \"/mfsdir1\"" actual &&
+    ipfs files rm -r --force /mfsdir1
+  '
+
+  test_expect_success "ipfs add file1 file2 --to-files /mfsdir2/ (with trailing slash) succeeds" '
+    ipfs files mkdir -p /mfsdir2 &&
+    test_expect_code 0 ipfs add --cid-version 1 test/mfs1.txt test/mfs2.txt --to-files /mfsdir2/ > actual 2>&1 &&
+    test_should_contain "added bafkreihm3rktn5z33luic3youqdsn326toaq3ekesmdvsa53sbrd3f5r3a mfs1.txt" actual &&
+    test_should_contain "added bafkreidh5zkhr2vnwa2luwmuj24xo6l3jhfgvkgtk5cyp43oxs7owzpxby mfs2.txt" actual &&
+    test_should_not_contain "Error" actual &&
+    ipfs files ls /mfsdir2/ > lsout &&
+    test_should_contain "mfs1.txt" lsout &&
+    test_should_contain "mfs2.txt" lsout &&
+    ipfs files rm -r --force /mfsdir2
+  '
+
+  test_expect_success "ipfs add file1 file2 --to-files /mfsfile2/ (with  trailing slash + with preexisting file) fails" '
+    echo test | ipfs files write --create /mfsfile2 &&
+    test_expect_code 1 ipfs add test/mfs1.txt test/mfs2.txt --to-files /mfsfile2/ >actual 2>&1 &&
+    test_should_contain "Error: to-files: MFS destination \"/mfsfile2/\" is not a directory" actual &&
+    ipfs files rm -r --force /mfsfile2
+  '
+
+  ## --to-files with recursive dir
+
+  # test MFS destination without trailing slash
+  test_expect_success "ipfs add with --to-files /mfs/subdir3 fails because /mfs/subdir3 exists" '
+    ipfs files mkdir -p /mfs/subdir3 &&
+    test_expect_code 1 ipfs add -r test --to-files /mfs/subdir3 >actual 2>&1 &&
+    test_should_contain "cannot put node in path \"/mfs/subdir3\": directory already has entry by that name" actual &&
+    ipfs files rm -r --force /mfs
+  '
+
+  # test recursive import of a dir  into MFS subdirectory
+  test_expect_success "ipfs add -r dir --to-files /mfs/subdir4/ succeeds (because of trailing slash)" '
+    ipfs files mkdir -p /mfs/subdir4 &&
+    ipfs add --cid-version 1 -r test --to-files /mfs/subdir4/ >actual 2>&1 &&
+    test_should_contain "added bafkreihm3rktn5z33luic3youqdsn326toaq3ekesmdvsa53sbrd3f5r3a test/mfs1.txt" actual &&
+    test_should_contain "added bafkreidh5zkhr2vnwa2luwmuj24xo6l3jhfgvkgtk5cyp43oxs7owzpxby test/mfs2.txt" actual &&
+    test_should_contain "added bafybeic7xwqwovt4g4bax6d3udp6222i63vj2rblpbim7uy2uw4a5gahha test" actual &&
+    test_should_not_contain "Error" actual
+    ipfs files ls /mfs/subdir4/ > lsout &&
+    test_should_contain "test" lsout &&
+    test_should_not_contain "mfs1.txt" lsout &&
+    test_should_not_contain "mfs2.txt" lsout &&
+    ipfs files rm -r --force /mfs
+  '
+
 }
 
 test_add_cat_5MB() {