Skip to content

Commit

Permalink
feat: dag import --stats (#8237)
Browse files Browse the repository at this point in the history
* feat: report block count on `dag import`
* fix: clean-up dag import message format
* Only print stats when --stats flag is passed

This applies to both text and json output encoding.

- Stats data is now contained within a Stats datastructure
- Stats are printed after root so that first line of output is the same as previously, even when stats are output using --stats

* fix sharness test

* Add PayloadBytesCount to stats

* Attempt to stabilize flaky tests

* Rename PayloadBytesCount to BlockBytesCount

* Correctly calculate size or imported dag

* Use RawSize of original block for import bytes calc

* test: dag import without --stats

basic regression tests for the default output (text and json)

Co-authored-by: gammazero <gammazero@users.noreply.github.com>
Co-authored-by: Marcin Rataj <lidel@lidel.org>
  • Loading branch information
3 people authored Sep 23, 2021
1 parent 6a10c1d commit 0057199
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 21 deletions.
36 changes: 31 additions & 5 deletions core/commands/dag/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ import (
)

const (
pinRootsOptionName = "pin-roots"
progressOptionName = "progress"
silentOptionName = "silent"
pinRootsOptionName = "pin-roots"
statsOptionName = "stats"
)

// DagCmd provides a subset of commands for interacting with ipld dag objects
Expand Down Expand Up @@ -53,9 +54,15 @@ type ResolveOutput struct {
RemPath string
}

type CarImportStats struct {
BlockCount uint64
BlockBytesCount uint64
}

// CarImportOutput is the output type of the 'dag import' commands
type CarImportOutput struct {
Root RootMeta
Root *RootMeta `json:",omitempty"`
Stats *CarImportStats `json:",omitempty"`
}

// RootMeta is the metadata for a root pinning response
Expand Down Expand Up @@ -160,8 +167,10 @@ var DagResolveCmd = &cmds.Command{
}

type importResult struct {
roots map[cid.Cid]struct{}
err error
blockCount uint64
blockBytesCount uint64
roots map[cid.Cid]struct{}
err error
}

// DagImportCmd is a command for importing a car to ipfs
Expand Down Expand Up @@ -193,8 +202,9 @@ Maximum supported CAR version: 1
cmds.FileArg("path", true, true, "The path of a .car file.").EnableStdin(),
},
Options: []cmds.Option{
cmds.BoolOption(silentOptionName, "No output."),
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
cmds.BoolOption(silentOptionName, "No output."),
cmds.BoolOption(statsOptionName, "Output stats."),
},
Type: CarImportOutput{},
Run: dagImport,
Expand All @@ -206,6 +216,22 @@ Maximum supported CAR version: 1
return nil
}

// event should have only one of `Root` or `Stats` set, not both
if event.Root == nil {
if event.Stats == nil {
return fmt.Errorf("Unexpected message from DAG import")
}
stats, _ := req.Options[statsOptionName].(bool)
if stats {
fmt.Fprintf(w, "Imported %d blocks (%d bytes)\n", event.Stats.BlockCount, event.Stats.BlockBytesCount)
}
return nil
}

if event.Stats != nil {
return fmt.Errorf("Unexpected message from DAG import")
}

enc, err := cmdenv.GetLowLevelCidEncoder(req)
if err != nil {
return err
Expand Down
23 changes: 21 additions & 2 deletions core/commands/dag/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
failedPins++
}

if err := res.Emit(&CarImportOutput{Root: ret}); err != nil {
if err := res.Emit(&CarImportOutput{Root: &ret}); err != nil {
return err
}
}
Expand All @@ -115,6 +115,19 @@ func dagImport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
}
}

stats, _ := req.Options[statsOptionName].(bool)
if stats {
err = res.Emit(&CarImportOutput{
Stats: &CarImportStats{
BlockCount: done.blockCount,
BlockBytesCount: done.blockBytesCount,
},
})
if err != nil {
return err
}
}

return nil
}

Expand All @@ -126,6 +139,7 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
batch := ipld.NewBatch(req.Context, api.Dag())

roots := make(map[cid.Cid]struct{})
var blockCount, blockBytesCount uint64

it := req.Files.Entries()
for it.Next() {
Expand Down Expand Up @@ -176,6 +190,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
if err := batch.Add(req.Context, nd); err != nil {
return err
}
blockCount++
blockBytesCount += uint64(len(block.RawData()))
}

return nil
Expand All @@ -197,5 +213,8 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
return
}

ret <- importResult{roots: roots}
ret <- importResult{
blockCount: blockCount,
blockBytesCount: blockBytesCount,
roots: roots}
}
2 changes: 1 addition & 1 deletion test/sharness/lib/test-lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ test_launch_ipfs_daemon() {

# wait for api file to show up
test_expect_success "api file shows up" '
test_wait_for_file 50 100ms "$IPFS_PATH/api"
test_wait_for_file 50 200ms "$IPFS_PATH/api"
'

test_set_address_vars actual_daemon
Expand Down
2 changes: 1 addition & 1 deletion test/sharness/t0041-ping.sh
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ test_expect_success "test ping 0" '
'

test_expect_success "test ping offline" '
iptb stop 1 &&
iptb stop 1 && sleep 2 &&
! ipfsi 0 ping -n2 -- "$PEERID_1"
'

Expand Down
56 changes: 44 additions & 12 deletions test/sharness/t0054-dag-car-import-export.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,18 +55,25 @@ run_online_imp_exp_tests() {
reset_blockstore 0
reset_blockstore 1

cat > basic_import_expected <<EOE
cat > basic_import_stats_expected <<EOE
Imported 1198 blocks (468513 bytes)
Pinned root${tab}bafkqaaa${tab}success
Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
EOE
# output without the --stats line at the top
tail -n +2 basic_import_stats_expected > basic_import_expected

# Explainer:
# naked_root_import_json_expected output is produced by dag import of combined_naked_roots_genesis_and_128.car
# executed when roots are already present in the repo - thus the BlockCount=0
# (if blocks were not present in the repo, blockstore: block not found would be returned)
cat >naked_root_import_json_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
{"Stats":{"BlockCount":0,"BlockBytesCount":0}}
EOE


test_expect_success "basic import" '
do_import 0 \
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
Expand All @@ -79,6 +86,18 @@ EOE
test_cmp_sorted basic_import_expected basic_import_actual
'

test_expect_success "basic import with --stats" '
do_import 0 --stats \
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car \
../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car \
> basic_import_actual
'

test_expect_success "basic import output with --stats as expected" '
test_cmp_sorted basic_import_stats_expected basic_import_actual
'

test_expect_success "basic fetch+export 1" '
ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
'
Expand All @@ -98,7 +117,7 @@ EOE
'

test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
ipfsi 1 dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
> naked_import_result_json_actual
'

Expand All @@ -117,7 +136,7 @@ EOE
cat ../t0054-dag-car-import-export-data/lotus_testnet_export_128_shuffled_nulroot.car > pipe_testnet &
cat ../t0054-dag-car-import-export-data/lotus_devnet_genesis_shuffled_nulroot.car > pipe_devnet &
do_import 0 \
do_import 0 --stats \
pipe_testnet \
pipe_devnet \
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
Expand All @@ -134,7 +153,7 @@ EOE
'

test_expect_success "fifo-import output as expected" '
test_cmp_sorted basic_import_expected basic_fifo_import_actual
test_cmp_sorted basic_import_stats_expected basic_fifo_import_actual
'
}

Expand Down Expand Up @@ -168,33 +187,46 @@ test_expect_success "correct error" '
test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
'


cat >multiroot_import_json_expected <<EOE
cat >multiroot_import_json_stats_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
{"Stats":{"BlockCount":2825,"BlockBytesCount":1339709}}
EOE
test_expect_success "multiroot import works" '
# output without --stats line
head -3 multiroot_import_json_stats_expected > multiroot_import_json_expected

test_expect_success "multiroot import works (--enc=json)" '
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
'
test_expect_success "multiroot import expected output" '
test_cmp_sorted multiroot_import_json_expected multiroot_import_json_actual
'

test_expect_success "multiroot import works with --stats" '
ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car > multiroot_import_json_actual
'
test_expect_success "multiroot import expected output" '
test_cmp_sorted multiroot_import_json_stats_expected multiroot_import_json_actual
'


cat >pin_import_expected << EOE
{"Stats":{"BlockCount":1198,"BlockBytesCount":468513}}
EOE
test_expect_success "pin-less import works" '
ipfs dag import --enc=json --pin-roots=false \
ipfs dag import --stats --enc=json --pin-roots=false \
../t0054-dag-car-import-export-data/lotus_devnet_genesis.car \
../t0054-dag-car-import-export-data/lotus_testnet_export_128.car \
> no-pin_import_actual
'
test_expect_success "expected silence on --pin-roots=false" '
test_cmp /dev/null no-pin_import_actual
test_expect_success "expected no pins on --pin-roots=false" '
test_cmp pin_import_expected no-pin_import_actual
'


test_expect_success "naked root import works" '
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
ipfs dag import --stats --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
> naked_root_import_json_actual
'
test_expect_success "naked root import expected output" '
Expand Down

0 comments on commit 0057199

Please sign in to comment.