From ab361e4747eb3c8a19bc76b042e0a6d63f4ec6b7 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Mon, 22 Apr 2024 21:29:25 +0700 Subject: [PATCH] move `temporal` package to erigon-lib (#10015) Co-authored-by: awskii --- cmd/evm/internal/t8ntool/transition.go | 4 +- cmd/integration/commands/root.go | 5 +- cmd/integration/commands/stages.go | 3 +- cmd/rpcdaemon/cli/config.go | 8 +- core/chain_makers.go | 10 +- core/genesis_test.go | 8 +- core/genesis_write.go | 4 +- core/rawdb/rawdbhelpers/rawdbhelpers.go | 4 +- core/rlp_test.go | 4 +- core/state/history_reader_v3.go | 4 +- core/state/rw_v3.go | 2 +- core/vm/gas_table_test.go | 4 +- erigon-lib/config3/config3.go | 4 + .../config3}/erigon3_test_disable.go | 2 +- .../config3}/erigon3_test_enable.go | 2 +- .../config3}/erigon4_test_enable.go | 2 +- .../kv}/temporal/kv_temporal.go | 252 ++++++------------ .../temporaltest/kv_temporal_testdb.go | 51 ++++ eth/backend.go | 8 +- eth/ethconfig/config.go | 2 - eth/stagedsync/default_stages.go | 14 +- eth/stagedsync/exec3.go | 8 +- eth/stagedsync/stage_call_traces_test.go | 4 +- eth/stagedsync/stage_execute.go | 3 +- eth/stagedsync/stage_execute_test.go | 10 +- eth/stagedsync/stage_hashstate_test.go | 16 +- eth/stagedsync/testutil.go | 4 +- p2p/sentry/sentry_grpc_server_test.go | 8 +- tests/block_test.go | 4 +- tests/exec_spec_test.go | 4 +- tests/state_test.go | 4 +- tests/state_test_util.go | 6 +- turbo/app/snapshots_cmd.go | 7 +- turbo/rpchelper/helper.go | 6 +- turbo/stages/genesis_test.go | 4 +- turbo/stages/mock/mock_sentry.go | 11 +- 36 files changed, 228 insertions(+), 268 deletions(-) create mode 100644 erigon-lib/config3/config3.go rename {eth/ethconfig => erigon-lib/config3}/erigon3_test_disable.go (86%) rename {eth/ethconfig => erigon-lib/config3}/erigon3_test_enable.go (84%) rename {eth/ethconfig => erigon-lib/config3}/erigon4_test_enable.go (84%) rename {core/state => erigon-lib/kv}/temporal/kv_temporal.go (64%) create mode 100644 erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 14401d0f94b..e1bd1710318 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -29,7 +29,7 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/common/datadir" - "github.com/ledgerwatch/erigon/core/state/temporal" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/log/v3" "github.com/urfave/cli/v2" @@ -294,7 +294,7 @@ func Main(ctx *cli.Context) error { return h } - _, db, _ := temporal.NewTestDB(nil, datadir.New(""), nil) + _, db, _ := temporaltest.NewTestDB(nil, datadir.New("")) defer db.Close() tx, err := db.BeginRw(context.Background()) diff --git a/cmd/integration/commands/root.go b/cmd/integration/commands/root.go index 95120c4f822..fcb9a932357 100644 --- a/cmd/integration/commands/root.go +++ b/cmd/integration/commands/root.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/ledgerwatch/erigon-lib/kv/temporal" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" @@ -16,8 +17,6 @@ import ( kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/erigon/cmd/utils" - "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/erigon/core/systemcontracts" "github.com/ledgerwatch/erigon/migrations" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/logging" @@ -106,7 +105,7 @@ func openDB(opts kv2.MdbxOpts, applyMigrations bool, logger log.Logger) (kv.RwDB } if h3 { _, _, agg := allSnapshots(context.Background(), db, logger) - tdb, err := temporal.New(db, agg, systemcontracts.SystemContractCodeLookup[chain]) + tdb, err := temporal.New(db, agg) if err != nil { return nil, err } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index cf7b514fdf5..5ed17e29a09 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -12,6 +12,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/erigontech/mdbx-go/mdbx" lru "github.com/hashicorp/golang-lru/arc/v2" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/secp256k1" "github.com/spf13/cobra" @@ -1459,7 +1460,7 @@ func allSnapshots(ctx context.Context, db kv.RoDB, logger log.Logger) (*freezebl _allBorSnapshotsSingleton = freezeblocks.NewBorRoSnapshots(snapCfg, dirs.Snap, 0, logger) var err error - _aggSingleton, err = libstate.NewAggregator(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db, logger) + _aggSingleton, err = libstate.NewAggregator(ctx, dirs.SnapHistory, dirs.Tmp, config3.HistoryV3AggregationStep, db, logger) if err != nil { panic(err) } diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 3f1be8d1bc7..5da0583753e 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/ledgerwatch/erigon-lib/config3" + "github.com/ledgerwatch/erigon-lib/kv/temporal" "github.com/ledgerwatch/log/v3" "github.com/spf13/cobra" "golang.org/x/sync/semaphore" @@ -51,8 +53,6 @@ import ( "github.com/ledgerwatch/erigon/consensus/ethash" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/erigon/core/systemcontracts" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/node" @@ -384,7 +384,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger allSnapshots.LogStat("remote") allBorSnapshots.LogStat("remote") - if agg, err = libstate.NewAggregator(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db, logger); err != nil { + if agg, err = libstate.NewAggregator(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, config3.HistoryV3AggregationStep, db, logger); err != nil { return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("create aggregator: %w", err) } _ = agg.OpenFolder() @@ -439,7 +439,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger }) if histV3Enabled { logger.Info("HistoryV3", "enable", histV3Enabled) - db, err = temporal.New(rwKv, agg, systemcontracts.SystemContractCodeLookup[cc.ChainName]) + db, err = temporal.New(rwKv, agg) if err != nil { return nil, nil, nil, nil, nil, nil, nil, nil, nil, err } diff --git a/core/chain_makers.go b/core/chain_makers.go index 78961f4beae..3b49adfec20 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -25,6 +25,7 @@ import ( "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/log/v3" @@ -34,7 +35,6 @@ import ( "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/trie" @@ -316,7 +316,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E var stateReader state.StateReader var stateWriter state.StateWriter - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") //agg := tx.(*temporal.Tx).Agg() //sd := agg.SharedDomains() @@ -332,7 +332,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E } txNum := -1 setBlockNum := func(blockNum uint64) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") //stateReader.(*state.StateReaderV4).SetBlockNum(blockNum) //stateWriter.(*state.StateWriterV4).SetBlockNum(blockNum) @@ -343,7 +343,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E } txNumIncrement := func() { txNum++ - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") //tx.(*temporal.Tx).Agg().SetTxNum(uint64(txNum)) //stateReader.(*state.StateReaderV4).SetTxNum(uint64(txNum)) @@ -386,7 +386,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E } var err error - b.header.Root, err = CalcHashRootForTests(tx, b.header, ethconfig.EnableHistoryV4InTest) + b.header.Root, err = CalcHashRootForTests(tx, b.header, config3.EnableHistoryV4InTest) if err != nil { return nil, nil, fmt.Errorf("call to CalcTrieRoot: %w", err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index 174a9df7c8b..16408207fa2 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -10,8 +10,8 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/erigon/core" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,7 +26,7 @@ import ( func TestGenesisBlockHashes(t *testing.T) { t.Parallel() logger := log.New() - _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, db, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) check := func(network string) { genesis := core.GenesisBlockByChainName(network) tx, err := db.BeginRw(context.Background()) @@ -77,7 +77,7 @@ func TestGenesisBlockRoots(t *testing.T) { func TestCommitGenesisIdempotency(t *testing.T) { t.Parallel() logger := log.New() - _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, db, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) tx, err := db.BeginRw(context.Background()) require.NoError(t, err) defer tx.Rollback() @@ -115,7 +115,7 @@ func TestAllocConstructor(t *testing.T) { }, } - historyV3, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + historyV3, db, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) _, _, err := core.CommitGenesisBlock(db, genSpec, "", logger) require.NoError(err) diff --git a/core/genesis_write.go b/core/genesis_write.go index e5aa29a33bd..505dd723b1d 100644 --- a/core/genesis_write.go +++ b/core/genesis_write.go @@ -28,6 +28,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" @@ -47,7 +48,6 @@ import ( "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/crypto" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/trie" ) @@ -191,7 +191,7 @@ func WriteGenesisState(g *types.Genesis, tx kv.RwTx, tmpDir string, logger log.L } var stateWriter state.StateWriter - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") //tx.(*temporal.Tx).Agg().SetTxNum(0) //stateWriter = state.NewWriterV4(tx.(kv.TemporalTx)) diff --git a/core/rawdb/rawdbhelpers/rawdbhelpers.go b/core/rawdb/rawdbhelpers/rawdbhelpers.go index 195c3950810..ee8ecd8c79e 100644 --- a/core/rawdb/rawdbhelpers/rawdbhelpers.go +++ b/core/rawdb/rawdbhelpers/rawdbhelpers.go @@ -3,8 +3,8 @@ package rawdbhelpers import ( "encoding/binary" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/erigon/eth/ethconfig" ) func IdxStepsCountV3(tx kv.Tx) float64 { @@ -14,7 +14,7 @@ func IdxStepsCountV3(tx kv.Tx) float64 { fstTxNum := binary.BigEndian.Uint64(fst) lstTxNum := binary.BigEndian.Uint64(lst) - return float64(lstTxNum-fstTxNum) / float64(ethconfig.HistoryV3AggregationStep) + return float64(lstTxNum-fstTxNum) / float64(config3.HistoryV3AggregationStep) } return 0 } diff --git a/core/rlp_test.go b/core/rlp_test.go index 0d03b0cdd75..cef7de0726f 100644 --- a/core/rlp_test.go +++ b/core/rlp_test.go @@ -24,7 +24,7 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" - "github.com/ledgerwatch/erigon/core/state/temporal" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/log/v3" "golang.org/x/crypto/sha3" @@ -37,7 +37,7 @@ import ( ) func getBlock(tb testing.TB, transactions int, uncles int, dataSize int, tmpDir string, logger log.Logger) *types.Block { - _, db, _ := temporal.NewTestDB(tb, datadir.New(tmpDir), nil) + _, db, _ := temporaltest.NewTestDB(tb, datadir.New(tmpDir)) var ( aa = libcommon.HexToAddress("0x000000000000000000000000000000000000aaaa") // Generate a canonical chain to act as the main dataset diff --git a/core/state/history_reader_v3.go b/core/state/history_reader_v3.go index 855d46d6d84..661b849aeb3 100644 --- a/core/state/history_reader_v3.go +++ b/core/state/history_reader_v3.go @@ -5,9 +5,9 @@ import ( "fmt" "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon/core/types/accounts" - "github.com/ledgerwatch/erigon/eth/ethconfig" ) // HistoryReaderV3 Implements StateReader and StateWriter @@ -51,7 +51,7 @@ func (hr *HistoryReaderV3) ReadAccountData(address common.Address) (*accounts.Ac func (hr *HistoryReaderV3) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { var acc []byte - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { acc = address.Bytes() } else { acc = make([]byte, 20+8) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index d3310e5b4f4..3428f6d08de 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -11,6 +11,7 @@ import ( "unsafe" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/kv/temporal" "github.com/ledgerwatch/log/v3" btree2 "github.com/tidwall/btree" @@ -24,7 +25,6 @@ import ( "github.com/ledgerwatch/erigon-lib/metrics" libstate "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/cmd/state/exec22" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/turbo/shards" ) diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 63c4546bd09..8f52ace30e3 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -31,7 +32,6 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/memdb" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/vm/evmtypes" "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/turbo/rpchelper" @@ -141,7 +141,7 @@ var createGasTests = []struct { func TestCreateGas(t *testing.T) { t.Parallel() - _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, db, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) for i, tt := range createGasTests { address := libcommon.BytesToAddress([]byte("contract")) diff --git a/erigon-lib/config3/config3.go b/erigon-lib/config3/config3.go new file mode 100644 index 00000000000..d371cafeafa --- /dev/null +++ b/erigon-lib/config3/config3.go @@ -0,0 +1,4 @@ +package config3 + +// AggregationStep number of transactions in smallest static file +const HistoryV3AggregationStep = 3_125_000 // 100M / 32 diff --git a/eth/ethconfig/erigon3_test_disable.go b/erigon-lib/config3/erigon3_test_disable.go similarity index 86% rename from eth/ethconfig/erigon3_test_disable.go rename to erigon-lib/config3/erigon3_test_disable.go index 77d80ace942..2857c940b11 100644 --- a/eth/ethconfig/erigon3_test_disable.go +++ b/erigon-lib/config3/erigon3_test_disable.go @@ -1,6 +1,6 @@ //go:build !erigon3 && !e3 && !erigon4 && !e4 -package ethconfig +package config3 const EnableHistoryV3InTest = false const EnableHistoryV4InTest = false diff --git a/eth/ethconfig/erigon3_test_enable.go b/erigon-lib/config3/erigon3_test_enable.go similarity index 84% rename from eth/ethconfig/erigon3_test_enable.go rename to erigon-lib/config3/erigon3_test_enable.go index ae1ee98d27a..39eb00b1fb7 100644 --- a/eth/ethconfig/erigon3_test_enable.go +++ b/erigon-lib/config3/erigon3_test_enable.go @@ -1,6 +1,6 @@ //go:build erigon3 || e3 -package ethconfig +package config3 const EnableHistoryV3InTest = true const EnableHistoryV4InTest = false diff --git a/eth/ethconfig/erigon4_test_enable.go b/erigon-lib/config3/erigon4_test_enable.go similarity index 84% rename from eth/ethconfig/erigon4_test_enable.go rename to erigon-lib/config3/erigon4_test_enable.go index 15d417ed914..2e6255a4067 100644 --- a/eth/ethconfig/erigon4_test_enable.go +++ b/erigon-lib/config3/erigon4_test_enable.go @@ -1,6 +1,6 @@ //go:build erigon4 || e4 -package ethconfig +package config3 const EnableHistoryV3InTest = true const EnableHistoryV4InTest = true diff --git a/core/state/temporal/kv_temporal.go b/erigon-lib/kv/temporal/kv_temporal.go similarity index 64% rename from core/state/temporal/kv_temporal.go rename to erigon-lib/kv/temporal/kv_temporal.go index 593c8bc5054..e6be7340fe6 100644 --- a/core/state/temporal/kv_temporal.go +++ b/erigon-lib/kv/temporal/kv_temporal.go @@ -2,28 +2,16 @@ package temporal import ( "context" - "encoding/binary" "fmt" - "testing" "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/datadir" - "github.com/ledgerwatch/erigon-lib/common/dir" - "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/iter" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/mdbx" - "github.com/ledgerwatch/erigon-lib/kv/memdb" "github.com/ledgerwatch/erigon-lib/kv/order" - "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/state" - "github.com/ledgerwatch/erigon/core/state/historyv2read" - "github.com/ledgerwatch/erigon/core/systemcontracts" - "github.com/ledgerwatch/erigon/core/types" - "github.com/ledgerwatch/erigon/core/types/accounts" - "github.com/ledgerwatch/erigon/eth/ethconfig" - "github.com/ledgerwatch/log/v3" ) //Variables Naming: @@ -59,47 +47,16 @@ import ( // HighLevel: // 1. Application - rely on TemporalDB (Ex: ExecutionLayer) or just DB (Ex: TxPool, Sentry, Downloader). -type tRestoreCodeHash func(tx kv.Getter, key, v []byte, force *common.Hash) ([]byte, error) -type tConvertAccount func(v []byte) ([]byte, error) -type tParseIncarnation func(v []byte) (uint64, error) - type DB struct { kv.RwDB agg *state.Aggregator - - convertV3toV2 tConvertAccount - convertV2toV3 tConvertAccount - restoreCodeHash tRestoreCodeHash - parseInc tParseIncarnation - systemContractLookup map[common.Address][]common.CodeRecord } -func New(db kv.RwDB, agg *state.Aggregator, systemContractLookup map[common.Address][]common.CodeRecord) (*DB, error) { +func New(db kv.RwDB, agg *state.Aggregator) (*DB, error) { if !kvcfg.HistoryV3.FromDB(db) { panic("not supported") } - if systemContractLookup != nil { - if err := db.View(context.Background(), func(tx kv.Tx) error { - var err error - for _, list := range systemContractLookup { - for i := range list { - list[i].TxNumber, err = rawdbv3.TxNums.Min(tx, list[i].BlockNumber) - if err != nil { - return err - } - } - } - return nil - }); err != nil { - return nil, err - } - } - - return &DB{RwDB: db, agg: agg, - convertV3toV2: accounts.ConvertV3toV2, convertV2toV3: accounts.ConvertV2toV3, - restoreCodeHash: historyv2read.RestoreCodeHash, parseInc: accounts.DecodeIncarnationFromStorage, - systemContractLookup: systemContractLookup, - }, nil + return &DB{RwDB: db, agg: agg}, nil } func (db *DB) Agg() *state.Aggregator { return db.agg } func (db *DB) InternalDB() kv.RwDB { return db.RwDB } @@ -225,26 +182,10 @@ func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, if len(v) == 0 { return k[:20], v, nil } - v, err = tx.db.convertV3toV2(v) - if err != nil { - return nil, nil, err - } - /* - var force *common.Hash - if tx.db.systemContractLookup != nil { - if records, ok := tx.db.systemContractLookup[common.BytesToAddress(k)]; ok { - p := sort.Search(len(records), func(i int) bool { - return records[i].TxNumber > asOfTs - }) - hash := records[p-1].CodeHash - force = &hash - } - } - v, err = tx.db.restoreCodeHash(tx.MdbxTx, k, v, force) - if err != nil { - return nil, nil, err - } - */ + //v, err = tx.db.convertV3toV2(v) + //if err != nil { + // return nil, nil, err + //} return k[:20], common.Copy(v), nil }) lastestStateIt, err := tx.RangeAscend(kv.PlainState, fromKey, toKey, -1) // don't apply limit, because need filter @@ -257,36 +198,36 @@ func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, }) it = iter.UnionKV(histStateIt2, latestStateIt2, limit) case kv.StorageDomain: - storageIt := tx.aggCtx.StorageHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx) - storageIt1 := iter.TransformKV(storageIt, func(k, v []byte) ([]byte, []byte, error) { - return k, v, nil - }) - - accData, err := tx.GetOne(kv.PlainState, fromKey[:20]) - if err != nil { - return nil, err - } - inc, err := tx.db.parseInc(accData) - if err != nil { - return nil, err - } - startkey := make([]byte, length.Addr+length.Incarnation+length.Hash) - copy(startkey, fromKey[:20]) - binary.BigEndian.PutUint64(startkey[length.Addr:], inc) - copy(startkey[length.Addr+length.Incarnation:], fromKey[20:]) - - toPrefix := make([]byte, length.Addr+length.Incarnation) - copy(toPrefix, fromKey[:20]) - binary.BigEndian.PutUint64(toPrefix[length.Addr:], inc+1) - - it2, err := tx.RangeAscend(kv.PlainState, startkey, toPrefix, limit) - if err != nil { - return nil, err - } - it3 := iter.TransformKV(it2, func(k, v []byte) ([]byte, []byte, error) { - return append(append([]byte{}, k[:20]...), k[28:]...), v, nil - }) - it = iter.UnionKV(storageIt1, it3, limit) + //storageIt := tx.aggCtx.StorageHistoricalStateRange(asOfTs, fromKey, toKey, limit, tx) + //storageIt1 := iter.TransformKV(storageIt, func(k, v []byte) ([]byte, []byte, error) { + // return k, v, nil + //}) + + //accData, err := tx.GetOne(kv.PlainState, fromKey[:20]) + //if err != nil { + // return nil, err + //} + //inc, err := tx.db.parseInc(accData) + //if err != nil { + // return nil, err + //} + //startkey := make([]byte, length.Addr+length.Incarnation+length.Hash) + //copy(startkey, fromKey[:20]) + //binary.BigEndian.PutUint64(startkey[length.Addr:], inc) + //copy(startkey[length.Addr+length.Incarnation:], fromKey[20:]) + // + //toPrefix := make([]byte, length.Addr+length.Incarnation) + //copy(toPrefix, fromKey[:20]) + //binary.BigEndian.PutUint64(toPrefix[length.Addr:], inc+1) + + //it2, err := tx.RangeAscend(kv.PlainState, startkey, toPrefix, limit) + //if err != nil { + // return nil, err + //} + //it3 := iter.TransformKV(it2, func(k, v []byte) ([]byte, []byte, error) { + // return append(append([]byte{}, k[:20]...), k[28:]...), v, nil + //}) + //it = iter.UnionKV(storageIt1, it3, limit) case kv.CodeDomain: panic("not implemented yet") default: @@ -300,7 +241,7 @@ func (tx *Tx) DomainRange(name kv.Domain, fromKey, toKey []byte, asOfTs uint64, return it, nil } func (tx *Tx) DomainGet(name kv.Domain, key, key2 []byte) (v []byte, ok bool, err error) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } switch name { @@ -318,49 +259,52 @@ func (tx *Tx) DomainGet(name kv.Domain, key, key2 []byte) (v []byte, ok bool, er } } func (tx *Tx) DomainGetAsOf(name kv.Domain, key, key2 []byte, ts uint64) (v []byte, ok bool, err error) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } - switch name { - case kv.AccountsDomain: - v, ok, err = tx.HistoryGet(kv.AccountsHistory, key, ts) - if err != nil { - return nil, false, err - } - if ok { - return v, true, nil - } - v, err = tx.GetOne(kv.PlainState, key) - if len(v) > 0 { - v, err = accounts.ConvertV2toV3(v) + /* + switch name { + case kv.AccountsDomain: + v, ok, err = tx.HistoryGet(kv.AccountsHistory, key, ts) if err != nil { return nil, false, err } + if ok { + return v, true, nil + } + v, err = tx.GetOne(kv.PlainState, key) + if len(v) > 0 { + v, err = accounts.ConvertV2toV3(v) + if err != nil { + return nil, false, err + } + } + return v, v != nil, err + case kv.StorageDomain: + v, ok, err = tx.HistoryGet(kv.StorageHistory, append(key[:20], key2...), ts) + if err != nil { + return nil, false, err + } + if ok { + return v, true, nil + } + v, err = tx.GetOne(kv.PlainState, append(key, key2...)) + return v, v != nil, err + case kv.CodeDomain: + v, ok, err = tx.HistoryGet(kv.CodeHistory, key, ts) + if err != nil { + return nil, false, err + } + if ok { + return v, true, nil + } + v, err = tx.GetOne(kv.Code, key2) + return v, v != nil, err + default: + panic(fmt.Sprintf("unexpected: %s", name)) } - return v, v != nil, err - case kv.StorageDomain: - v, ok, err = tx.HistoryGet(kv.StorageHistory, append(key[:20], key2...), ts) - if err != nil { - return nil, false, err - } - if ok { - return v, true, nil - } - v, err = tx.GetOne(kv.PlainState, append(key, key2...)) - return v, v != nil, err - case kv.CodeDomain: - v, ok, err = tx.HistoryGet(kv.CodeHistory, key, ts) - if err != nil { - return nil, false, err - } - if ok { - return v, true, nil - } - v, err = tx.GetOne(kv.Code, key2) - return v, v != nil, err - default: - panic(fmt.Sprintf("unexpected: %s", name)) - } + */ + panic("not implemented yet") } func (tx *Tx) HistoryGet(name kv.History, key []byte, ts uint64) (v []byte, ok bool, err error) { @@ -445,43 +389,3 @@ func (tx *Tx) HistoryRange(name kv.History, fromTs, toTs int, asc order.By, limi } return it, err } - -// TODO: need remove `gspec` param (move SystemContractCodeLookup feature somewhere) -func NewTestDB(tb testing.TB, dirs datadir.Dirs, gspec *types.Genesis) (histV3 bool, db kv.RwDB, agg *state.Aggregator) { - historyV3 := ethconfig.EnableHistoryV3InTest - logger := log.New() - ctx := context.Background() - - if tb != nil { - db = memdb.NewTestDB(tb) - } else { - db = memdb.New(dirs.DataDir) - } - _ = db.UpdateNosync(context.Background(), func(tx kv.RwTx) error { - _, _ = kvcfg.HistoryV3.WriteOnce(tx, historyV3) - return nil - }) - - if historyV3 { - var err error - dir.MustExist(dirs.SnapHistory) - agg, err = state.NewAggregator(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db, logger) - if err != nil { - panic(err) - } - if err := agg.OpenFolder(); err != nil { - panic(err) - } - - var sc map[common.Address][]common.CodeRecord - if gspec != nil { - sc = systemcontracts.SystemContractCodeLookup[gspec.Config.ChainName] - } - - db, err = New(db, agg, sc) - if err != nil { - panic(err) - } - } - return historyV3, db, agg -} diff --git a/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go new file mode 100644 index 00000000000..baab298e2d4 --- /dev/null +++ b/erigon-lib/kv/temporal/temporaltest/kv_temporal_testdb.go @@ -0,0 +1,51 @@ +package temporaltest + +import ( + "context" + "testing" + + "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon-lib/common/dir" + "github.com/ledgerwatch/erigon-lib/config3" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon-lib/kv/temporal" + "github.com/ledgerwatch/erigon-lib/state" + "github.com/ledgerwatch/log/v3" +) + +//nolint:thelper +func NewTestDB(tb testing.TB, dirs datadir.Dirs) (histV3 bool, db kv.RwDB, agg *state.Aggregator) { + historyV3 := config3.EnableHistoryV3InTest + logger := log.New() + ctx := context.Background() + + if tb != nil { + db = memdb.NewTestDB(tb) + } else { + db = memdb.New(dirs.DataDir) + } + _ = db.UpdateNosync(context.Background(), func(tx kv.RwTx) error { + _, _ = kvcfg.HistoryV3.WriteOnce(tx, historyV3) + return nil + }) + + if historyV3 { + var err error + dir.MustExist(dirs.SnapHistory) + agg, err = state.NewAggregator(ctx, dirs.SnapHistory, dirs.Tmp, config3.HistoryV3AggregationStep, db, logger) + if err != nil { + panic(err) + } + if err := agg.OpenFolder(); err != nil { + panic(err) + } + + db, err = temporal.New(db, agg) + if err != nil { + panic(err) + } + } + return historyV3, db, agg +} diff --git a/eth/backend.go b/eth/backend.go index 9839c78f22e..2acd655f9e5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -36,6 +36,8 @@ import ( "github.com/erigontech/mdbx-go/mdbx" lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/config3" + "github.com/ledgerwatch/erigon-lib/kv/temporal" "github.com/ledgerwatch/log/v3" "golang.org/x/exp/slices" "google.golang.org/grpc" @@ -87,8 +89,6 @@ import ( "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/rawdb/blockio" - "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/erigon/core/systemcontracts" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/crypto" @@ -334,7 +334,7 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger backend.agg, backend.blockSnapshots, backend.blockReader, backend.blockWriter = agg, allSnapshots, blockReader, blockWriter if config.HistoryV3 { - backend.chainDB, err = temporal.New(backend.chainDB, agg, systemcontracts.SystemContractCodeLookup[config.Genesis.Config.ChainName]) + backend.chainDB, err = temporal.New(backend.chainDB, agg) if err != nil { return nil, err } @@ -1344,7 +1344,7 @@ func setUpBlockReader(ctx context.Context, db kv.RwDB, dirs datadir.Dirs, snConf blockReader := freezeblocks.NewBlockReader(allSnapshots, allBorSnapshots) blockWriter := blockio.NewBlockWriter(histV3) - agg, err := libstate.NewAggregator(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db, logger) + agg, err := libstate.NewAggregator(ctx, dirs.SnapHistory, dirs.Tmp, config3.HistoryV3AggregationStep, db, logger) if err != nil { return nil, nil, nil, nil, nil, err } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index c36288e6686..e1e0efad543 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -45,8 +45,6 @@ import ( "github.com/ledgerwatch/erigon/rpc" ) -// AggregationStep number of transactions in smallest static file -const HistoryV3AggregationStep = 3_125_000 // 100M / 32 //const HistoryV3AggregationStep = 3_125_000 / 100 // use this to reduce step size for dev/debug // FullNodeGPO contains default gasprice oracle settings for full node. diff --git a/eth/stagedsync/default_stages.go b/eth/stagedsync/default_stages.go index 08dfdd8430e..6473732be4e 100644 --- a/eth/stagedsync/default_stages.go +++ b/eth/stagedsync/default_stages.go @@ -3,12 +3,12 @@ package stagedsync import ( "context" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/wrap" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" ) @@ -133,7 +133,7 @@ func DefaultStages(ctx context.Context, { ID: stages.HashState, Description: "Hash the key in the state", - Disabled: bodies.historyV3 || ethconfig.EnableHistoryV4InTest || dbg.StagesOnlyBlocks, + Disabled: bodies.historyV3 || config3.EnableHistoryV4InTest || dbg.StagesOnlyBlocks, Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger) }, @@ -147,7 +147,7 @@ func DefaultStages(ctx context.Context, { ID: stages.IntermediateHashes, Description: "Generate intermediate hashes and computing state root", - Disabled: bodies.historyV3 || ethconfig.EnableHistoryV4InTest || dbg.StagesOnlyBlocks, + Disabled: bodies.historyV3 || config3.EnableHistoryV4InTest || dbg.StagesOnlyBlocks, Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { if exec.chainConfig.IsOsaka(0) { _, err := SpawnVerkleTrie(s, u, txc.Tx, trieCfg, ctx, logger) @@ -313,7 +313,7 @@ func PipelineStages(ctx context.Context, snapshots SnapshotsCfg, blockHashCfg Bl { ID: stages.HashState, Description: "Hash the key in the state", - Disabled: exec.historyV3 && ethconfig.EnableHistoryV4InTest, + Disabled: exec.historyV3 && config3.EnableHistoryV4InTest, Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger) }, @@ -327,7 +327,7 @@ func PipelineStages(ctx context.Context, snapshots SnapshotsCfg, blockHashCfg Bl { ID: stages.IntermediateHashes, Description: "Generate intermediate hashes and computing state root", - Disabled: exec.historyV3 && ethconfig.EnableHistoryV4InTest, + Disabled: exec.historyV3 && config3.EnableHistoryV4InTest, Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { if exec.chainConfig.IsOsaka(0) { _, err := SpawnVerkleTrie(s, u, txc.Tx, trieCfg, ctx, logger) @@ -522,7 +522,7 @@ func UploaderPipelineStages(ctx context.Context, snapshots SnapshotsCfg, headers { ID: stages.HashState, Description: "Hash the key in the state", - Disabled: exec.historyV3 && ethconfig.EnableHistoryV4InTest, + Disabled: exec.historyV3 && config3.EnableHistoryV4InTest, Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { return SpawnHashStateStage(s, txc.Tx, hashState, ctx, logger) }, @@ -536,7 +536,7 @@ func UploaderPipelineStages(ctx context.Context, snapshots SnapshotsCfg, headers { ID: stages.IntermediateHashes, Description: "Generate intermediate hashes and computing state root", - Disabled: exec.historyV3 && ethconfig.EnableHistoryV4InTest, + Disabled: exec.historyV3 && config3.EnableHistoryV4InTest, Forward: func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { if exec.chainConfig.IsOsaka(0) { _, err := SpawnVerkleTrie(s, u, txc.Tx, trieCfg, ctx, logger) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 72b20d7e436..b90124c1ceb 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -15,6 +15,7 @@ import ( "github.com/c2h5oh/datasize" "github.com/erigontech/mdbx-go/mdbx" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "golang.org/x/sync/errgroup" @@ -38,7 +39,6 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb/rawdbhelpers" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/turbo/services" @@ -88,7 +88,7 @@ func (p *Progress) Log(rs *state.StateV3, in *exec22.QueueWithRetry, rws *exec22 "buffer", fmt.Sprintf("%s/%s", common.ByteCount(sizeEstimate), common.ByteCount(p.commitThreshold)), "idxStepsInDB", fmt.Sprintf("%.2f", idxStepsAmountInDB), //"inBlk", inputBlockNum, - "step", fmt.Sprintf("%.1f", float64(outTxNum)/float64(ethconfig.HistoryV3AggregationStep)), + "step", fmt.Sprintf("%.1f", float64(outTxNum)/float64(config3.HistoryV3AggregationStep)), "alloc", common.ByteCount(m.Alloc), "sys", common.ByteCount(m.Sys), ) //var txNums []string @@ -342,7 +342,7 @@ func ExecV3(ctx context.Context, case <-pruneEvery.C: if rs.SizeEstimate() < commitThreshold { if agg.CanPrune(tx) { - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep*10); err != nil { // prune part of retired data, before commit + if err = agg.Prune(ctx, config3.HistoryV3AggregationStep*10); err != nil { // prune part of retired data, before commit return err } } else { @@ -473,7 +473,7 @@ func ExecV3(ctx context.Context, if block < cfg.blockReader.FrozenBlocks() { agg.KeepInDB(0) - defer agg.KeepInDB(ethconfig.HistoryV3AggregationStep) + defer agg.KeepInDB(config3.HistoryV3AggregationStep) } getHeaderFunc := func(hash common.Hash, number uint64) (h *types.Header) { diff --git a/eth/stagedsync/stage_call_traces_test.go b/eth/stagedsync/stage_call_traces_test.go index 0248d65ae47..539601a661a 100644 --- a/eth/stagedsync/stage_call_traces_test.go +++ b/eth/stagedsync/stage_call_traces_test.go @@ -10,9 +10,9 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/stretchr/testify/require" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/log/v3" ) @@ -35,7 +35,7 @@ func genTestCallTraceSet(t *testing.T, tx kv.RwTx, to uint64) { func TestCallTrace(t *testing.T) { logger := log.New() ctx, require := context.Background(), require.New(t) - histV3, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + histV3, db, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) if histV3 { t.Skip() } diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index 9ed00465420..3480ed7d2da 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -10,6 +10,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" "golang.org/x/sync/errgroup" @@ -946,7 +947,7 @@ func PruneExecutionStage(s *PruneState, tx kv.RwTx, cfg ExecuteBlockCfg, ctx con if cfg.historyV3 { cfg.agg.SetTx(tx) if initialCycle { - if err = cfg.agg.Prune(ctx, ethconfig.HistoryV3AggregationStep/10); err != nil { // prune part of retired data, before commit + if err = cfg.agg.Prune(ctx, config3.HistoryV3AggregationStep/10); err != nil { // prune part of retired data, before commit return err } } else { diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go index 279c758aef0..37621dadb13 100644 --- a/eth/stagedsync/stage_execute_test.go +++ b/eth/stagedsync/stage_execute_test.go @@ -8,6 +8,8 @@ import ( "time" "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon-lib/config3" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/require" @@ -21,8 +23,6 @@ import ( "github.com/ledgerwatch/erigon/cmd/state/exec22" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/erigon/ethdb/prune" "github.com/ledgerwatch/erigon/params" @@ -173,7 +173,7 @@ func apply(tx kv.RwTx, agg *libstate.Aggregator, logger log.Logger) (beforeBlock func newAgg(t *testing.T, logger log.Logger) *libstate.Aggregator { t.Helper() dir, ctx := t.TempDir(), context.Background() - agg, err := libstate.NewAggregator(ctx, dir, dir, ethconfig.HistoryV3AggregationStep, nil, logger) + agg, err := libstate.NewAggregator(ctx, dir, dir, config3.HistoryV3AggregationStep, nil, logger) require.NoError(t, err) err = agg.OpenFolder() require.NoError(t, err) @@ -183,8 +183,8 @@ func newAgg(t *testing.T, logger log.Logger) *libstate.Aggregator { func TestExec22(t *testing.T) { logger := log.New() ctx := context.Background() - _, db1, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) - _, db2, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, db1, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + _, db2, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) agg := newAgg(t, logger) cfg := ExecuteBlockCfg{historyV3: true, agg: agg} diff --git a/eth/stagedsync/stage_hashstate_test.go b/eth/stagedsync/stage_hashstate_test.go index 81d5c12206a..fc6619121a3 100644 --- a/eth/stagedsync/stage_hashstate_test.go +++ b/eth/stagedsync/stage_hashstate_test.go @@ -6,9 +6,9 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/memdb" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/stretchr/testify/require" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -16,7 +16,7 @@ import ( ) func TestPromoteHashedStateClearState(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } logger := log.New() @@ -37,7 +37,7 @@ func TestPromoteHashedStateClearState(t *testing.T) { } func TestPromoteHashedStateIncremental(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } logger := log.New() @@ -67,7 +67,7 @@ func TestPromoteHashedStateIncremental(t *testing.T) { } func TestPromoteHashedStateIncrementalMixed(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } logger := log.New() @@ -88,7 +88,7 @@ func TestPromoteHashedStateIncrementalMixed(t *testing.T) { } func TestUnwindHashed(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } logger := log.New() @@ -115,7 +115,7 @@ func TestUnwindHashed(t *testing.T) { } func TestPromoteIncrementallyShutdown(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } historyV3 := false @@ -150,7 +150,7 @@ func TestPromoteIncrementallyShutdown(t *testing.T) { } func TestPromoteHashedStateCleanlyShutdown(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } logger := log.New() @@ -189,7 +189,7 @@ func TestPromoteHashedStateCleanlyShutdown(t *testing.T) { } func TestUnwindHashStateShutdown(t *testing.T) { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { t.Skip() } logger := log.New() diff --git a/eth/stagedsync/testutil.go b/eth/stagedsync/testutil.go index feea1059193..545e0c631b0 100644 --- a/eth/stagedsync/testutil.go +++ b/eth/stagedsync/testutil.go @@ -7,11 +7,11 @@ import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" state2 "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/types/accounts" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/stretchr/testify/assert" ) @@ -29,7 +29,7 @@ func compareCurrentState( buckets ...string, ) { for _, bucket := range buckets { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { compareDomain(t, agg, db1, db2, bucket) continue } diff --git a/p2p/sentry/sentry_grpc_server_test.go b/p2p/sentry/sentry_grpc_server_test.go index 5cd8739c9a8..37d296f5b1a 100644 --- a/p2p/sentry/sentry_grpc_server_test.go +++ b/p2p/sentry/sentry_grpc_server_test.go @@ -13,13 +13,13 @@ import ( "github.com/ledgerwatch/erigon-lib/gointerfaces" proto_sentry "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/stretchr/testify/require" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/forkid" "github.com/ledgerwatch/erigon/core/rawdb" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/p2p" "github.com/ledgerwatch/log/v3" @@ -83,8 +83,8 @@ func testForkIDSplit(t *testing.T, protocol uint) { SpuriousDragonBlock: big.NewInt(2), ByzantiumBlock: big.NewInt(3), } - _, dbNoFork, _ = temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) - _, dbProFork, _ = temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, dbNoFork, _ = temporaltest.NewTestDB(t, datadir.New(t.TempDir())) + _, dbProFork, _ = temporaltest.NewTestDB(t, datadir.New(t.TempDir())) gspecNoFork = &types.Genesis{Config: configNoFork} gspecProFork = &types.Genesis{Config: configProFork} @@ -176,7 +176,7 @@ func TestSentryServerImpl_SetStatusInitPanic(t *testing.T) { }() configNoFork := &chain.Config{HomesteadBlock: big.NewInt(1), ChainID: big.NewInt(1)} - _, dbNoFork, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, dbNoFork, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) gspecNoFork := &types.Genesis{Config: configNoFork} genesisNoFork := core.MustCommitGenesis(gspecNoFork, dbNoFork, "", log.Root()) ss := &GrpcServer{p2p: &p2p.Config{}} diff --git a/tests/block_test.go b/tests/block_test.go index f5851c11bc8..6807efc67fb 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -22,7 +22,7 @@ import ( "runtime" "testing" - "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" ) @@ -42,7 +42,7 @@ func TestBlockchain(t *testing.T) { // TODO(yperbasis): make it work bt.skipLoad(`^TransitionTests/bcArrowGlacierToMerge/powToPosBlockRejection\.json`) bt.skipLoad(`^TransitionTests/bcFrontierToHomestead/blockChainFrontierWithLargerTDvsHomesteadBlockchain\.json`) - if ethconfig.EnableHistoryV3InTest { + if config3.EnableHistoryV3InTest { // HistoryV3: doesn't produce receipts on execution by design bt.skipLoad(`^InvalidBlocks/bcInvalidHeaderTest/log1_wrongBloom\.json`) bt.skipLoad(`^InvalidBlocks/bcInvalidHeaderTest/wrongReceiptTrie\.json`) diff --git a/tests/exec_spec_test.go b/tests/exec_spec_test.go index de67b0a6315..c996b73268c 100644 --- a/tests/exec_spec_test.go +++ b/tests/exec_spec_test.go @@ -6,12 +6,12 @@ import ( "path/filepath" "testing" - "github.com/ledgerwatch/erigon/eth/ethconfig" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/log/v3" ) func TestExecutionSpec(t *testing.T) { - if ethconfig.EnableHistoryV3InTest { + if config3.EnableHistoryV3InTest { t.Skip("fix me in e3 please") } diff --git a/tests/state_test.go b/tests/state_test.go index 844288364c2..0dfb32a8f14 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -28,7 +28,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/common/datadir" - "github.com/ledgerwatch/erigon/core/state/temporal" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/core/vm" @@ -49,7 +49,7 @@ func TestState(t *testing.T) { st.skipLoad(`^stTimeConsuming/`) st.skipLoad(`.*vmPerformance/loop.*`) - _, db, _ := temporal.NewTestDB(t, datadir.New(t.TempDir()), nil) + _, db, _ := temporaltest.NewTestDB(t, datadir.New(t.TempDir())) st.walk(t, stateTestDir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { subtest := subtest diff --git a/tests/state_test_util.go b/tests/state_test_util.go index b1691e23b54..1ebf7b1a99d 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -26,6 +26,7 @@ import ( "strings" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/config3" "golang.org/x/crypto/sha3" "github.com/ledgerwatch/erigon-lib/chain" @@ -42,7 +43,6 @@ import ( "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/crypto" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/rlp" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/trie" @@ -200,7 +200,7 @@ func (t *StateTest) RunNoVerify(tx kv.RwTx, subtest StateSubtest, vmconfig vm.Co statedb := state.New(r) var w state.StateWriter - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } else { w = state.NewPlainStateWriter(tx, nil, writeBlockNr) @@ -336,7 +336,7 @@ func MakePreState(rules *chain.Rules, tx kv.RwTx, accounts types.GenesisAlloc, b } var w state.StateWriter - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } else { w = state.NewPlainStateWriter(tx, nil, blockNr+1) diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index 07d0f8ecb7f..5acf0333ff8 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -15,6 +15,7 @@ import ( "time" "github.com/c2h5oh/datasize" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon/cl/clparams" "github.com/ledgerwatch/log/v3" "github.com/urfave/cli/v2" @@ -615,7 +616,7 @@ func doRetireCommand(cliCtx *cli.Context) error { for i := 0; i < 1024; i++ { if err := db.UpdateNosync(ctx, func(tx kv.RwTx) error { agg.SetTx(tx) - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep/2); err != nil { + if err = agg.Prune(ctx, config3.HistoryV3AggregationStep/2); err != nil { return err } return err @@ -661,7 +662,7 @@ func doRetireCommand(cliCtx *cli.Context) error { for i := 0; i < 1024; i++ { if err := db.UpdateNosync(ctx, func(tx kv.RwTx) error { agg.SetTx(tx) - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep/10); err != nil { + if err = agg.Prune(ctx, config3.HistoryV3AggregationStep/10); err != nil { return err } return err @@ -817,7 +818,7 @@ func dbCfg(label kv.Label, path string) mdbx.MdbxOpts { return opts } func openAgg(ctx context.Context, dirs datadir.Dirs, chainDB kv.RwDB, logger log.Logger) *libstate.Aggregator { - agg, err := libstate.NewAggregator(ctx, dirs.Snap, dirs.Tmp, ethconfig.HistoryV3AggregationStep, chainDB, logger) + agg, err := libstate.NewAggregator(ctx, dirs.Snap, dirs.Tmp, config3.HistoryV3AggregationStep, chainDB, logger) if err != nil { panic(err) } diff --git a/turbo/rpchelper/helper.go b/turbo/rpchelper/helper.go index 2a6ccc0544b..75404b8cff7 100644 --- a/turbo/rpchelper/helper.go +++ b/turbo/rpchelper/helper.go @@ -6,13 +6,13 @@ import ( "fmt" libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/config3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/systemcontracts" - "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" borfinality "github.com/ledgerwatch/erigon/polygon/bor/finality" "github.com/ledgerwatch/erigon/polygon/bor/finality/whitelist" @@ -146,14 +146,14 @@ func CreateHistoryStateReader(tx kv.Tx, blockNumber uint64, txnIndex int, histor } func NewLatestStateReader(tx kv.Getter) state.StateReader { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") //b.pendingReader = state.NewReaderV4(b.pendingReaderTx.(kv.TemporalTx)) } return state.NewPlainStateReader(tx) } func NewLatestStateWriter(tx kv.RwTx, blockNum uint64) state.StateWriter { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } return state.NewPlainStateWriter(tx, tx, blockNum) diff --git a/turbo/stages/genesis_test.go b/turbo/stages/genesis_test.go index 578ea96bba7..2d5642bddfd 100644 --- a/turbo/stages/genesis_test.go +++ b/turbo/stages/genesis_test.go @@ -27,8 +27,8 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/erigon/core" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/eth/ethconfig" @@ -176,7 +176,7 @@ func TestSetupGenesis(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() dirs := datadir.New(tmpdir) - _, db, _ := temporal.NewTestDB(t, dirs, nil) + _, db, _ := temporaltest.NewTestDB(t, dirs) blockReader := freezeblocks.NewBlockReader(freezeblocks.NewRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, dirs.Snap, 0, log.New()), freezeblocks.NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: false}, dirs.Snap, 0, log.New())) config, genesis, err := test.fn(db) // Check the return values. diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 146b41ae1ac..4d87a5567a9 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -13,6 +13,8 @@ import ( "github.com/c2h5oh/datasize" lru "github.com/hashicorp/golang-lru/arc/v2" "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/config3" + "github.com/ledgerwatch/erigon-lib/kv/temporal/temporaltest" "github.com/ledgerwatch/log/v3" "google.golang.org/protobuf/types/known/emptypb" @@ -40,7 +42,6 @@ import ( "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/rawdb/blockio" "github.com/ledgerwatch/erigon/core/state" - "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" "github.com/ledgerwatch/erigon/crypto" @@ -256,7 +257,7 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK logger := log.New() ctx, ctxCancel := context.WithCancel(context.Background()) - histV3, db, agg := temporal.NewTestDB(nil, dirs, nil) + histV3, db, agg := temporaltest.NewTestDB(nil, dirs) cfg.HistoryV3 = histV3 erigonGrpcServeer := remotedbserver.NewKvServer(ctx, db, nil, nil, nil, logger) @@ -777,21 +778,21 @@ func (ms *MockSentry) NewHistoryStateReader(blockNum uint64, tx kv.Tx) state.Sta } func (ms *MockSentry) NewStateReader(tx kv.Tx) state.StateReader { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } return state.NewPlainStateReader(tx) } func (ms *MockSentry) NewStateWriter(tx kv.RwTx, blockNum uint64) state.StateWriter { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") } return state.NewPlainStateWriter(tx, tx, blockNum) } func (ms *MockSentry) CalcStateRoot(tx kv.Tx) libcommon.Hash { - if ethconfig.EnableHistoryV4InTest { + if config3.EnableHistoryV4InTest { panic("implement me") }