Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

cmd/integration: print_table_sizes #10061

Merged
merged 4 commits into from
Apr 25, 2024
Merged

cmd/integration: print_table_sizes #10061

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ae26cd4
cmd/integration: print_table_sizes
taratorio Apr 25, 2024
6319619
cmd/integration: csv output to console
taratorio Apr 25, 2024
14b2432
cmd/integration: print table sizes add output csv file
taratorio Apr 25, 2024
02dccad
cmd/integration: print table sizes to use db.AllTables
taratorio Apr 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 48 additions & 1 deletion cmd/integration/commands/stages.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/c2h5oh/datasize"
"github.com/erigontech/mdbx-go/mdbx"
lru "github.com/hashicorp/golang-lru/arc/v2"
"github.com/ledgerwatch/erigon-lib/config3"
"github.com/ledgerwatch/log/v3"
"github.com/ledgerwatch/secp256k1"
"github.com/spf13/cobra"
Expand All @@ -24,6 +23,7 @@ import (
"github.com/ledgerwatch/erigon-lib/common/cmp"
"github.com/ledgerwatch/erigon-lib/common/datadir"
"github.com/ledgerwatch/erigon-lib/common/dir"
"github.com/ledgerwatch/erigon-lib/config3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/kvcfg"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
Expand Down Expand Up @@ -313,6 +313,7 @@ var cmdStageTxLookup = &cobra.Command{
}
},
}

var cmdPrintStages = &cobra.Command{
Use: "print_stages",
Short: "",
Expand All @@ -334,6 +335,49 @@ var cmdPrintStages = &cobra.Command{
},
}

var cmdPrintTableSizes = &cobra.Command{
Use: "print_table_sizes",
Short: "",
Run: func(cmd *cobra.Command, args []string) {
logger := debug.SetupCobra(cmd, "integration")
db, err := openDB(dbCfg(kv.ChainDB, chaindata), false, logger)
if err != nil {
logger.Error("Opening DB", "error", err)
return
}
defer db.Close()

var tableSizes []interface{}
err = db.View(cmd.Context(), func(tx kv.Tx) error {
tableSizes = stagedsync.CollectTableSizes(db, tx, kv.ChaindataTables)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can use db.AllTables() instead of kv.ChaindataTables

Copy link
Member Author

@taratorio taratorio Apr 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nice, thanks - added it and also an --output.csv.file arg

return nil
})
if err != nil {
logger.Error("error while collecting table sizes", "err", err)
return
}

if len(tableSizes)%2 != 0 {
logger.Error("table sizes len not even", "len", len(tableSizes))
return
}

var sb strings.Builder
sb.WriteString("Table")
sb.WriteRune(',')
sb.WriteString("Size")
sb.WriteRune('\n')
for i := 0; i < len(tableSizes)/2; i++ {
sb.WriteString(tableSizes[i*2].(string))
sb.WriteRune(',')
sb.WriteString(tableSizes[i*2+1].(string))
sb.WriteRune('\n')
}

fmt.Print(sb.String())
},
}

var cmdPrintMigrations = &cobra.Command{
Use: "print_migrations",
Short: "",
Expand Down Expand Up @@ -476,6 +520,9 @@ func init() {
withHeimdall(cmdPrintStages)
rootCmd.AddCommand(cmdPrintStages)

withDataDir(cmdPrintTableSizes)
rootCmd.AddCommand(cmdPrintTableSizes)

withConfig(cmdStageSenders)
withIntegrityChecks(cmdStageSenders)
withReset(cmdStageSenders)
Expand Down
19 changes: 13 additions & 6 deletions eth/stagedsync/sync.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,16 +462,23 @@ func (s *Sync) PrintTimings() []interface{} {
return logCtx
}

func PrintTables(db kv.RoDB, tx kv.RwTx) []interface{} {
if tx == nil {
return nil
}
buckets := []string{
func CollectDBMetrics(db kv.RoDB, tx kv.RwTx) []interface{} {
res := CollectTableSizes(db, tx, []string{
kv.PlainState,
kv.AccountChangeSet,
kv.StorageChangeSet,
kv.EthTx,
kv.Log,
})

tx.CollectMetrics()

return res
}

func CollectTableSizes(db kv.RoDB, tx kv.Tx, buckets []string) []interface{} {
if tx == nil {
return nil
}
bucketSizes := make([]interface{}, 0, 2*(len(buckets)+2))
for _, bucket := range buckets {
Expand All @@ -491,7 +498,7 @@ func PrintTables(db kv.RoDB, tx kv.RwTx) []interface{} {
if db != nil {
bucketSizes = append(bucketSizes, "ReclaimableSpace", libcommon.ByteCount(amountOfFreePagesInDb*db.PageSize()))
}
tx.CollectMetrics()

return bucketSizes
}

Expand Down
2 changes: 1 addition & 1 deletion turbo/stages/stageloop.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ func StageLoopIteration(ctx context.Context, db kv.RwDB, txc wrap.TxContainer, s
var tableSizes []interface{}
var commitTime time.Duration
if canRunCycleInOneTransaction && !externalTx {
tableSizes = stagedsync.PrintTables(db, txc.Tx) // Need to do this before commit to access tx
tableSizes = stagedsync.CollectDBMetrics(db, txc.Tx) // Need to do this before commit to access tx
commitStart := time.Now()
errTx := txc.Tx.Commit()
txc.Tx = nil
Expand Down
Loading