From ae26cd4b27edce541cea57f3dd0ec5dd31e1aefa Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:22:31 +0300 Subject: [PATCH 1/4] cmd/integration: print_table_sizes --- cmd/integration/commands/stages.go | 39 +++++++++++++++++++++++++++++- eth/stagedsync/sync.go | 19 ++++++++++----- turbo/stages/stageloop.go | 2 +- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index f13a18a6581..e4213a680fe 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -12,7 +12,6 @@ import ( "github.com/c2h5oh/datasize" "github.com/erigontech/mdbx-go/mdbx" lru "github.com/hashicorp/golang-lru/arc/v2" - "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/secp256k1" "github.com/spf13/cobra" @@ -24,6 +23,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" @@ -313,6 +313,7 @@ var cmdStageTxLookup = &cobra.Command{ } }, } + var cmdPrintStages = &cobra.Command{ Use: "print_stages", Short: "", @@ -334,6 +335,39 @@ var cmdPrintStages = &cobra.Command{ }, } +var cmdPrintTableSizes = &cobra.Command{ + Use: "print_table_sizes", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + logger := debug.SetupCobra(cmd, "integration") + db, err := openDB(dbCfg(kv.ChainDB, chaindata), false, logger) + if err != nil { + logger.Error("Opening DB", "error", err) + return + } + defer db.Close() + + var tableSizes []interface{} + err = db.View(cmd.Context(), func(tx kv.Tx) error { + tableSizes = stagedsync.CollectTableSizes(db, tx, kv.ChaindataTables) + return nil + }) + if err != nil { + logger.Error("error while collecting table sizes", "err", err) + return + } + + if len(tableSizes)%2 != 0 { + logger.Error("table sizes len not even", "len", len(tableSizes)) + return + } + + for i := 0; i < len(tableSizes)/2; i++ { + logger.Info(fmt.Sprintf("%d", i+1), tableSizes[i*2], tableSizes[i*2+1]) + } + }, +} + var cmdPrintMigrations = &cobra.Command{ Use: "print_migrations", Short: "", @@ -476,6 +510,9 @@ func init() { withHeimdall(cmdPrintStages) rootCmd.AddCommand(cmdPrintStages) + withDataDir(cmdPrintTableSizes) + rootCmd.AddCommand(cmdPrintTableSizes) + withConfig(cmdStageSenders) withIntegrityChecks(cmdStageSenders) withReset(cmdStageSenders) diff --git a/eth/stagedsync/sync.go b/eth/stagedsync/sync.go index 8af2d280283..a0a8dc366b3 100644 --- a/eth/stagedsync/sync.go +++ b/eth/stagedsync/sync.go @@ -462,16 +462,23 @@ func (s *Sync) PrintTimings() []interface{} { return logCtx } -func PrintTables(db kv.RoDB, tx kv.RwTx) []interface{} { - if tx == nil { - return nil - } - buckets := []string{ +func CollectDBMetrics(db kv.RoDB, tx kv.RwTx) []interface{} { + res := CollectTableSizes(db, tx, []string{ kv.PlainState, kv.AccountChangeSet, kv.StorageChangeSet, kv.EthTx, kv.Log, + }) + + tx.CollectMetrics() + + return res +} + +func CollectTableSizes(db kv.RoDB, tx kv.Tx, buckets []string) []interface{} { + if tx == nil { + return nil } bucketSizes := make([]interface{}, 0, 2*(len(buckets)+2)) for _, bucket := range buckets { @@ -491,7 +498,7 @@ func PrintTables(db kv.RoDB, tx kv.RwTx) []interface{} { if db != nil { bucketSizes = append(bucketSizes, "ReclaimableSpace", libcommon.ByteCount(amountOfFreePagesInDb*db.PageSize())) } - tx.CollectMetrics() + return bucketSizes } diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 01fee094183..1365df8a317 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -154,7 +154,7 @@ func StageLoopIteration(ctx context.Context, db kv.RwDB, txc wrap.TxContainer, s var tableSizes []interface{} var commitTime time.Duration if canRunCycleInOneTransaction && !externalTx { - tableSizes = stagedsync.PrintTables(db, txc.Tx) // Need to do this before commit to access tx + tableSizes = stagedsync.CollectDBMetrics(db, txc.Tx) // Need to do this before commit to access tx commitStart := time.Now() errTx := txc.Tx.Commit() txc.Tx = nil From 6319619ee67ca7aa11d8dd2127fd9f087e1861cb Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:44:38 +0300 Subject: [PATCH 2/4] cmd/integration: csv output to console --- cmd/integration/commands/stages.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index e4213a680fe..72a7cdf595c 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -362,9 +362,19 @@ var cmdPrintTableSizes = &cobra.Command{ return } + var sb strings.Builder + sb.WriteString("Table") + sb.WriteRune(',') + sb.WriteString("Size") + sb.WriteRune('\n') for i := 0; i < len(tableSizes)/2; i++ { - logger.Info(fmt.Sprintf("%d", i+1), tableSizes[i*2], tableSizes[i*2+1]) + sb.WriteString(tableSizes[i*2].(string)) + sb.WriteRune(',') + sb.WriteString(tableSizes[i*2+1].(string)) + sb.WriteRune('\n') } + + fmt.Print(sb.String()) }, } From 14b243207bb12331cb874c382f2341082d8595cd Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:00:55 +0300 Subject: [PATCH 3/4] cmd/integration: print table sizes add output csv file --- cmd/integration/commands/flags.go | 5 +++++ cmd/integration/commands/stages.go | 21 ++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/cmd/integration/commands/flags.go b/cmd/integration/commands/flags.go index bf57f18a61c..a9927ae12b9 100644 --- a/cmd/integration/commands/flags.go +++ b/cmd/integration/commands/flags.go @@ -31,6 +31,7 @@ var ( pruneTBefore, pruneCBefore uint64 experiments []string chain string // Which chain to use (mainnet, goerli, sepolia, etc.) + outputCsvFile string commitmentMode string commitmentTrie string @@ -169,6 +170,10 @@ func withTraceFromTx(cmd *cobra.Command) { cmd.Flags().Uint64Var(&traceFromTx, "txtrace.from", 0, "start tracing from tx number") } +func withOutputCsvFile(cmd *cobra.Command) { + cmd.Flags().StringVar(&outputCsvFile, "output.csv.file", "", "location to output csv data") +} + func withCommitment(cmd *cobra.Command) { cmd.Flags().StringVar(&commitmentMode, "commitment.mode", "direct", "defines the way to calculate commitments: 'direct' mode reads from state directly, 'update' accumulate updates before commitment, 'off' actually disables commitment calculation") cmd.Flags().StringVar(&commitmentTrie, "commitment.trie", "hex", "hex - use Hex Patricia Hashed Trie for commitments, bin - use of binary patricia trie") diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 72a7cdf595c..a5c12866e10 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "os" "strings" "sync" "time" @@ -374,7 +375,24 @@ var cmdPrintTableSizes = &cobra.Command{ sb.WriteRune('\n') } - fmt.Print(sb.String()) + if outputCsvFile == "" { + logger.Info("table sizes", "csv", sb.String()) + return + } + + f, err := os.Create(outputCsvFile) + if err != nil { + logger.Error("issue creating file", "file", outputCsvFile, "err", err) + return + } + + _, err = f.WriteString(sb.String()) + if err != nil { + logger.Error("issue writing output to file", "file", outputCsvFile, "err", err) + return + } + + logger.Info("wrote table sizes to csv output file", "file", outputCsvFile) }, } @@ -521,6 +539,7 @@ func init() { rootCmd.AddCommand(cmdPrintStages) withDataDir(cmdPrintTableSizes) + withOutputCsvFile(cmdPrintTableSizes) rootCmd.AddCommand(cmdPrintTableSizes) withConfig(cmdStageSenders) From 02dccad83163726d6dfb18b6ae0a4bbe11dfa0cc Mon Sep 17 00:00:00 2001 From: taratorio <94537774+taratorio@users.noreply.github.com> Date: Thu, 25 Apr 2024 15:03:51 +0300 Subject: [PATCH 4/4] cmd/integration: print table sizes to use db.AllTables --- cmd/integration/commands/stages.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index a5c12866e10..f265fd35983 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -348,9 +348,15 @@ var cmdPrintTableSizes = &cobra.Command{ } defer db.Close() + allTablesCfg := db.AllTables() + allTables := make([]string, 0, len(allTablesCfg)) + for table := range allTablesCfg { + allTables = append(allTables, table) + } + var tableSizes []interface{} err = db.View(cmd.Context(), func(tx kv.Tx) error { - tableSizes = stagedsync.CollectTableSizes(db, tx, kv.ChaindataTables) + tableSizes = stagedsync.CollectTableSizes(db, tx, allTables) return nil }) if err != nil {