diff --git a/.gitattributes b/.gitattributes index d6b38c11dd0..831606f194f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -9,6 +9,8 @@ LICENSE text eol=auto *.png binary *.tar binary *.gz binary +*.xz binary +*.car binary # Binary assets assets/init-doc/* binary diff --git a/.gitignore b/.gitignore index 41f66ac9110..90109ade45a 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ vendor .tarball go-ipfs-source.tar.gz docs/examples/go-ipfs-as-a-library/example-folder/Qm* +/test/sharness/t0054-dag-car-import-export-data/*.car diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index e34c5070c94..7c8065bb32d 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -95,7 +95,9 @@ func TestCommands(t *testing.T) { "/config/profile/apply", "/dag", "/dag/get", + "/dag/export", "/dag/put", + "/dag/import", "/dag/resolve", "/dht", "/dht/findpeer", diff --git a/core/commands/dag/dag.go b/core/commands/dag/dag.go index 66d50feb1f5..88f6ebadc9d 100644 --- a/core/commands/dag/dag.go +++ b/core/commands/dag/dag.go @@ -1,29 +1,48 @@ package dagcmd import ( + "errors" "fmt" "io" "math" + "os" "strings" + "time" "github.com/ipfs/go-ipfs/core/commands/cmdenv" "github.com/ipfs/go-ipfs/core/coredag" + iface "github.com/ipfs/interface-go-ipfs-core" cid "github.com/ipfs/go-cid" cidenc "github.com/ipfs/go-cidutil/cidenc" cmds "github.com/ipfs/go-ipfs-cmds" files "github.com/ipfs/go-ipfs-files" ipld "github.com/ipfs/go-ipld-format" + mdag "github.com/ipfs/go-merkledag" ipfspath "github.com/ipfs/go-path" + "github.com/ipfs/interface-go-ipfs-core/options" path "github.com/ipfs/interface-go-ipfs-core/path" mh "github.com/multiformats/go-multihash" + + gocar "github.com/ipld/go-car" + //gipfree "github.com/ipld/go-ipld-prime/impl/free" + //gipselector "github.com/ipld/go-ipld-prime/traversal/selector" + //gipselectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder" + + "gopkg.in/cheggaaa/pb.v1" +) + +const ( + progressOptionName = "progress" + silentOptionName = "silent" + pinRootsOptionName = "pin-roots" ) var DagCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Interact with ipld dag objects.", ShortDescription: ` -'ipfs dag' is used for creating and manipulating dag objects. +'ipfs dag' is used for creating and manipulating dag objects/hierarchies. This subcommand is currently an experimental feature, but it is intended to deprecate and replace the existing 'ipfs object' command moving forward. @@ -33,6 +52,8 @@ to deprecate and replace the existing 'ipfs object' command moving forward. "put": DagPutCmd, "get": DagGetCmd, "resolve": DagResolveCmd, + "import": DagImportCmd, + "export": DagExportCmd, }, } @@ -47,6 +68,15 @@ type ResolveOutput struct { RemPath string } +// CarImportOutput is the output type of the 'dag import' commands +type CarImportOutput struct { + Root RootMeta +} +type RootMeta struct { + Cid cid.Cid + PinErrorMsg string +} + var DagPutCmd = &cmds.Command{ Helptext: cmds.HelpText{ Tagline: "Add a dag node to ipfs.", @@ -241,3 +271,400 @@ var DagResolveCmd = &cmds.Command{ }, Type: ResolveOutput{}, } + +type importResult struct { + roots map[cid.Cid]struct{} + err error +} + +var DagImportCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Import the contents of .car files", + ShortDescription: ` +'ipfs dag import' imports all blocks present in supplied .car +( Content Address aRchive ) files, recursively pinning any roots +specified in the CAR file headers, unless --pin-roots is set to false. + +Note: + This command will import all blocks in the CAR file, not just those + reachable from the specified roots. However, these other blocks will + not be pinned and may be garbage collected later. + + The pinning of the roots happens after all car files are processed, + permitting import of DAGs spanning multiple files. + + Pinning takes place in offline-mode exclusively, one root at a time. + If the combination of blocks from the imported CAR files and what is + currently present in the blockstore does not represent a complete DAG, + pinning of that individual root will fail. + +Maximum supported CAR version: 1 +`, + }, + Arguments: []cmds.Argument{ + cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.BoolOption(silentOptionName, "No output."), + cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true), + }, + Type: CarImportOutput{}, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + api, err := cmdenv.GetApi(env, req) + if err != nil { + return err + } + + // on import ensure we do not reach out to the network for any reason + // if a pin based on what is imported + what is in the blockstore + // isn't possible: tough luck + api, err = api.WithOptions(options.Api.Offline(true)) + if err != nil { + return err + } + + // grab a pinlock ( which doubles as a GC lock ) so that regardless of the + // size of the streamed-in cars nothing will disappear on us before we had + // a chance to roots that may show up at the very end + // This is especially important for use cases like dagger: + // ipfs dag import $( ... | ipfs-dagger --stdout=carfifos ) + // + unlocker := node.Blockstore.PinLock() + defer unlocker.Unlock() + + doPinRoots, _ := req.Options[pinRootsOptionName].(bool) + + retCh := make(chan importResult, 1) + go importWorker(req, res, api, retCh) + + done := <-retCh + if done.err != nil { + return done.err + } + + // It is not guaranteed that a root in a header is actually present in the same ( or any ) + // .car file. This is the case in version 1, and ideally in further versions too + // Accumulate any root CID seen in a header, and supplement its actual node if/when encountered + // We will attempt a pin *only* at the end in case all car files were well formed + // + // The boolean value indicates whether we have encountered the root within the car file's + roots := done.roots + + // opportunistic pinning: try whatever sticks + if doPinRoots { + + var failedPins int + for c := range roots { + + // We need to re-retrieve a block, convert it to ipld, and feed it + // to the Pinning interface, sigh... + // + // If we didn't have the problem of inability to take multiple pinlocks, + // we could use the Api directly like so (though internally it does the same): + // + // // not ideal, but the pinning api takes only paths :( + // rp := path.NewResolvedPath( + // ipfspath.FromCid(c), + // c, + // c, + // "", + // ) + // + // if err := api.Pin().Add(req.Context, rp, options.Pin.Recursive(true)); err != nil { + + ret := RootMeta{Cid: c} + + if block, err := node.Blockstore.Get(c); err != nil { + ret.PinErrorMsg = err.Error() + } else if nd, err := ipld.Decode(block); err != nil { + ret.PinErrorMsg = err.Error() + } else if err := node.Pinning.Pin(req.Context, nd, true); err != nil { + ret.PinErrorMsg = err.Error() + } else if err := node.Pinning.Flush(req.Context); err != nil { + ret.PinErrorMsg = err.Error() + } + + if ret.PinErrorMsg != "" { + failedPins++ + } + + if err := res.Emit(&CarImportOutput{Root: ret}); err != nil { + return err + } + } + + if failedPins > 0 { + return fmt.Errorf( + "unable to pin all roots: %d out of %d failed", + failedPins, + len(roots), + ) + } + } + + return nil + }, + Encoders: cmds.EncoderMap{ + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, event *CarImportOutput) error { + + silent, _ := req.Options[silentOptionName].(bool) + if silent { + return nil + } + + enc, err := cmdenv.GetLowLevelCidEncoder(req) + if err != nil { + return err + } + + if event.Root.PinErrorMsg != "" { + event.Root.PinErrorMsg = fmt.Sprintf("FAILED: %s", event.Root.PinErrorMsg) + } else { + event.Root.PinErrorMsg = "success" + } + + _, err = fmt.Fprintf( + w, + "Pinned root\t%s\t%s\n", + enc.Encode(event.Root.Cid), + event.Root.PinErrorMsg, + ) + return err + }), + }, +} + +func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI, ret chan importResult) { + + // this is *not* a transaction + // it is simply a way to relieve pressure on the blockstore + // similar to pinner.Pin/pinner.Flush + batch := ipld.NewBatch(req.Context, api.Dag()) + + roots := make(map[cid.Cid]struct{}) + + it := req.Files.Entries() + for it.Next() { + + file := files.FileFromEntry(it) + if file == nil { + ret <- importResult{err: errors.New("expected a file handle")} + return + } + + // wrap a defer-closer-scope + // + // every single file in it() is already open before we start + // just close here sooner rather than later for neatness + // and to surface potential erorrs writing on closed fifos + // this won't/can't help with not running out of handles + err := func() error { + defer file.Close() + + car, err := gocar.NewCarReader(file) + if err != nil { + return err + } + + // Be explicit here, until the spec is finished + if car.Header.Version != 1 { + return errors.New("only car files version 1 supported at present") + } + + for _, c := range car.Header.Roots { + roots[c] = struct{}{} + } + + for { + block, err := car.Next() + if err != nil && err != io.EOF { + return err + } else if block == nil { + break + } + + // the double-decode is suboptimal, but we need it for batching + nd, err := ipld.Decode(block) + if err != nil { + return err + } + + if err := batch.Add(req.Context, nd); err != nil { + return err + } + } + + return nil + }() + + if err != nil { + ret <- importResult{err: err} + return + } + } + + if err := it.Err(); err != nil { + ret <- importResult{err: err} + return + } + + if err := batch.Commit(); err != nil { + ret <- importResult{err: err} + return + } + + ret <- importResult{roots: roots} +} + +var DagExportCmd = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Streams the selected DAG as a .car stream on stdout.", + ShortDescription: ` +'ipfs dag export' fetches a dag and streams it out as a well-formed .car file. +Note that at present only single root selections / .car files are supported. +The output of blocks happens in strict DAG-traversal, first-seen, order. +`, + }, + Arguments: []cmds.Argument{ + cmds.StringArg("root", true, false, "CID of a root to recursively export").EnableStdin(), + }, + Options: []cmds.Option{ + cmds.BoolOption(progressOptionName, "p", "Display progress on CLI. Defaults to true when STDERR is a TTY."), + }, + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + + c, err := cid.Decode(req.Arguments[0]) + if err != nil { + return fmt.Errorf( + "unable to parse root specification (currently only bare CIDs are supported): %s", + err, + ) + } + + node, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + // Code disabled until descent-issue in go-ipld-prime is fixed + // https://github.com/ribasushi/gip-muddle-up + // + // sb := gipselectorbuilder.NewSelectorSpecBuilder(gipfree.NodeBuilder()) + // car := gocar.NewSelectiveCar( + // req.Context, + // , + // []gocar.Dag{gocar.Dag{ + // Root: c, + // Selector: sb.ExploreRecursive( + // gipselector.RecursionLimitNone(), + // sb.ExploreAll(sb.ExploreRecursiveEdge()), + // ).Node(), + // }}, + // ) + // ... + // if err := car.Write(pipeW); err != nil {} + + pipeR, pipeW := io.Pipe() + + errCh := make(chan error, 2) // we only report the 1st error + go func() { + defer func() { + if err := pipeW.Close(); err != nil { + errCh <- fmt.Errorf("stream flush failed: %s", err) + } + close(errCh) + }() + + if err := gocar.WriteCar( + req.Context, + mdag.NewSession( + req.Context, + node.DAG, + ), + []cid.Cid{c}, + pipeW, + ); err != nil { + errCh <- err + } + }() + + if err := res.Emit(pipeR); err != nil { + pipeR.Close() // ignore the error if any + return err + } + + err = <-errCh + + // minimal user friendliness + if err != nil && + !node.IsOnline && + err == ipld.ErrNotFound { + err = fmt.Errorf("%s (currently offline, perhaps retry after attaching to the network)", err) + } + + return err + }, + PostRun: cmds.PostRunMap{ + cmds.CLI: func(res cmds.Response, re cmds.ResponseEmitter) error { + + var showProgress bool + val, specified := res.Request().Options[progressOptionName] + if !specified { + // default based on TTY availability + errStat, _ := os.Stderr.Stat() + if 0 != (errStat.Mode() & os.ModeCharDevice) { + showProgress = true + } + } else if val.(bool) { + showProgress = true + } + + // simple passthrough, no progress + if !showProgress { + return cmds.Copy(re, res) + } + + bar := pb.New64(0).SetUnits(pb.U_BYTES) + bar.Output = os.Stderr + bar.ShowSpeed = true + bar.ShowElapsedTime = true + bar.RefreshRate = 500 * time.Millisecond + bar.Start() + + var processedOneResponse bool + for { + v, err := res.Next() + if err == io.EOF { + + // We only write the final bar update on success + // On error it looks too weird + bar.Finish() + + return re.Close() + } else if err != nil { + return re.CloseWithError(err) + } else if processedOneResponse { + return re.CloseWithError(errors.New("unexpected multipart response during emit, please file a bugreport")) + } + + r, ok := v.(io.Reader) + if !ok { + // some sort of encoded response, this should not be happening + return errors.New("unexpected non-stream passed to PostRun: please file a bugreport") + } + + processedOneResponse = true + + if err := re.Emit(bar.NewProxyReader(r)); err != nil { + return err + } + } + }, + }, +} diff --git a/go.mod b/go.mod index fab16101adb..a6d52b0cfab 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/ipfs/go-ipfs-routing v0.1.0 github.com/ipfs/go-ipfs-util v0.0.1 github.com/ipfs/go-ipld-cbor v0.0.4 - github.com/ipfs/go-ipld-format v0.0.2 + github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-ipld-git v0.0.3 github.com/ipfs/go-ipns v0.0.2 github.com/ipfs/go-log v1.0.3 @@ -54,6 +54,7 @@ require ( github.com/ipfs/go-unixfs v0.2.4 github.com/ipfs/go-verifcid v0.0.1 github.com/ipfs/interface-go-ipfs-core v0.2.6 + github.com/ipld/go-car v0.1.0 github.com/jbenet/go-is-domain v1.0.3 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/jbenet/go-temp-err-catcher v0.1.0 diff --git a/go.sum b/go.sum index 0e3cbf12895..7d836da7f6d 100644 --- a/go.sum +++ b/go.sum @@ -303,6 +303,8 @@ github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9 github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= github.com/ipfs/go-ipld-git v0.0.3 h1:/YjkjCyo5KYRpW+suby8Xh9Cm/iH9dAgGV6qyZ1dGus= github.com/ipfs/go-ipld-git v0.0.3/go.mod h1:RuvMXa9qtJpDbqngyICCU/d+cmLFXxLsbIclmD0Lcr0= github.com/ipfs/go-ipns v0.0.2 h1:oq4ErrV4hNQ2Eim257RTYRgfOSV/s8BDaf9iIl4NwFs= @@ -326,6 +328,7 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0= github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4= github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ= @@ -354,6 +357,8 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/interface-go-ipfs-core v0.2.6 h1:4eeGPJUDWblEurSzpAtL2znDG1xqAoX2aNNIoOFwyuc= github.com/ipfs/interface-go-ipfs-core v0.2.6/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s= +github.com/ipld/go-car v0.1.0 h1:AaIEA5ITRnFA68uMyuIPYGM2XXllxsu8sNjFJP797us= +github.com/ipld/go-car v0.1.0/go.mod h1:RCWzaUh2i4mOEkB3W45Vc+9jnS/M6Qay5ooytiBHl3g= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4= diff --git a/test/sharness/t0054-dag-car-import-export-data/README.md b/test/sharness/t0054-dag-car-import-export-data/README.md new file mode 100644 index 00000000000..77679af390f --- /dev/null +++ b/test/sharness/t0054-dag-car-import-export-data/README.md @@ -0,0 +1,24 @@ +# Dataset description/sources + +- lotus_testnet_export_256_multiroot.car + - Export of the first 256 block of the testnet chain, with 3 tipset roots. Exported from Lotus by @traviperson on 2019-03-18 + + +- lotus_devnet_genesis.car + - Source: https://github.com/filecoin-project/lotus/blob/v0.2.10/build/genesis/devnet.car + +- lotus_testnet_export_128.car + - Export of the first 128 block of the testnet chain, exported from Lotus by @traviperson on 2019-03-24 + + +- lotus_devnet_genesis_shuffled_noroots.car +- lotus_testnet_export_128_shuffled_noroots.car + - versions of the above with an **empty** root array, and having all blocks shuffled + +- lotus_devnet_genesis_shuffled_nulroot.car +- lotus_testnet_export_128_shuffled_nulroot.car + - versions identical to the above, but with a single "empty-block" root each ( in order to work around go-car not following the current "roots can be empty" spec ) + +- combined_naked_roots_genesis_and_128.car + - only the roots of `lotus_devnet_genesis.car` and `lotus_testnet_export_128.car`, to to be used in combination with the root-less parts to validate "transactional" pinning + diff --git a/test/sharness/t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz b/test/sharness/t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz new file mode 100644 index 00000000000..34eb36dd1ca Binary files /dev/null and b/test/sharness/t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz differ diff --git a/test/sharness/t0054-dag-car-import-export.sh b/test/sharness/t0054-dag-car-import-export.sh new file mode 100755 index 00000000000..33652b2aa6f --- /dev/null +++ b/test/sharness/t0054-dag-car-import-export.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# + +test_description="Test car file import/export functionality" + +. lib/test-lib.sh +export -f ipfsi + +set -o pipefail + +tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz + +reset_blockstore() { + node=$1 + ipfsi $1 pin ls --quiet --type=recursive | ipfsi $1 pin rm &>/dev/null + ipfsi $1 repo gc &>/dev/null + + test_expect_success "pinlist empty" ' + [ "$( ipfsi $1 pin ls )" = "" ] + ' + test_expect_success "nothing left to gc" ' + [ "$( ipfsi $1 repo gc )" = "" ] + ' +} + +# hammer with concurrent gc to ensure nothing clashes +do_import() { + node=$1; shift + + bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc1_pid=$! + bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc2_pid=$! + + ipfsi $node dag import "$@" + + rm -f spin.gc || true + sleep 3 + kill $gc1_pid $gc2_pid || true +} + +run_online_imp_exp_tests() { + + reset_blockstore 0 + reset_blockstore 1 + + echo -e "Pinned root\tbafkqaaa\tsuccess" > basic_import_expected + echo -e "Pinned root\tbafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u\tsuccess" >> basic_import_expected + echo -e "Pinned root\tbafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy\tsuccess" >> basic_import_expected + + touch spin.gc + test_expect_success "basic import" ' + do_import 0 \ + ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \ + ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \ + | sort > basic_import_actual + ' + + # FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed + test_expect_failure "concurrent GC did not manage to grab anything and remained silent" ' + ! [[ -s gc_out ]] + ' + test_expect_success "basic import output as expected" ' + test_cmp basic_import_expected basic_import_actual + ' + + reset_blockstore 0 + reset_blockstore 1 + + mkfifo pipe_testnet + mkfifo pipe_devnet + + # test that ipfs correctly opens both pipes and deleting them doesn't interfere with cleanup + bash -c ' + sleep 1 + cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet & cat1_pid=$! + cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet & cat2_pid=$! + + rm pipe_testnet pipe_devnet + + # extra safety valve to kill the cat processes in case something goes wrong + bash -c "sleep 60; kill $cat1_pid $cat2_pid 2>/dev/null" & + ' & + + touch spin.gc + test_expect_success "fifo import" ' + do_import 0 \ + pipe_testnet \ + pipe_devnet \ + ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \ + | sort > basic_fifo_import_actual + ' + # FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed + test_expect_failure "concurrent GC did not manage to grab anything and remained silent" ' + ! [[ -s gc_out ]] + ' + + test_expect_success "fifo-import output as expected" ' + test_cmp basic_import_expected basic_fifo_import_actual + ' + + test_expect_success "fifos no longer present" ' + ! [[ -e pipe_testnet ]] && ! [[ -e pipe_devnet ]] + ' +} + + +test_expect_success "set up testbed" ' + iptb testbed create -type localipfs -count 2 -force -init +' +startup_cluster 2 + +run_online_imp_exp_tests + +test_expect_success "shut down nodes" ' + iptb stop && iptb_wait_stop +' + + +# We want to just init the repo, without using a daemon for stuff below +test_init_ipfs + + +test_expect_success "basic offline export of 'getting started' dag works" ' + ipfs dag export QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv >/dev/null +' + + +echo "Error: merkledag: not found (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected +test_expect_success "basic offline export of nonexistent cid" ' + ! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null +' +test_expect_success "correct error" ' + test_cmp offline_fetch_error_expected offline_fetch_error_actual +' + + +cat >multiroot_import_expected < multiroot_import_actual +' +test_expect_success "multiroot import expected output" ' + test_cmp multiroot_import_expected multiroot_import_actual +' + + +test_expect_success "pin-less import works" ' + ipfs dag import --enc=json --pin-roots=false \ + ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \ + ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \ + > no-pin_import_actual +' +test_expect_success "expected silence on --pin-roots=false" ' + ! [[ -s no-pin_import_actual ]] +' + + +cat >naked_root_import_expected < naked_root_import_actual +' +test_expect_success "naked root import expected output" ' + test_cmp naked_root_import_expected naked_root_import_actual +' + + +test_done