diff --git a/README.md b/README.md index 8ee63718b93..0031b5e1ac9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,45 @@ +# Otterscan v2.3.0-alpha enabled Erigon + +This branch contains the experimental version of Erigon with Otterscan API **v2.3.0-alpha**. + +Compatibility: + +- Erigon v2.55.1 +- Otterscan v2.3.0-alpha + +> This version is compatible with v2.0.0-alpha. If you are currently running it, you can just replace the binary as there were no DB model breaking changes. New DB tables will be populated during the first run though. + +## Warnings/disclaimers (READ THIS FIRST!!!) + +This is an experimental build. Make sure you are aware of the following first: + +- Don't use this build on production servers. +- Make sure you backup your Erigon node before trying this build. +- This version is made available in source-code form only. +- DB model will change on next alphas. There will **NOT** be migration scripts. You'll need to restore your original Erigon node from a backup. +- For now it is compatible with Erigon 2 only, but the end game is to support Erigon 3 only. At some point in the future we may change it. +- Enabling Otterscan v2 support will produce extra data inside Erigon DB, so you'll need more disk space than a regular Erigon node. +- At the first run of this patched build, extra stages will produce new indexes from genesis, so there will be extra sync time. Once it reaches the tip, the extra stages will take neglegible time per block iteration. +- **Alphas were not optimized for space/time (yet). The goal here was to implement the spec for all token/contract indexing support and prove it was doable.** + +## How to use it + +### Erigon + +Build this branch as usual with `make` command. + +Add the `--experimental.ots2` CLI argument to the `erigon` command. That'll enable Otterscan V2 extra stages. + +Also enable the `ots2` API namespace, i.e., `--http.api "eth,erigon,ots,ots2"`. + +### Otterscan + +See: https://github.com/otterscan/otterscan/blob/develop/docs/ots2.md + +> The rest of this document contains the original Erigon README. + +--- + # Erigon Erigon is an implementation of Ethereum (execution layer with embeddable consensus layer), on the efficiency diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 18959df6224..d15e5b10b43 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -10,6 +10,7 @@ import ( "sync" "time" + "github.com/RoaringBitmap/roaring/roaring64" "github.com/c2h5oh/datasize" "github.com/erigontech/mdbx-go/mdbx" lru "github.com/hashicorp/golang-lru/arc/v2" @@ -29,6 +30,7 @@ import ( chain2 "github.com/ledgerwatch/erigon-lib/chain" "github.com/ledgerwatch/erigon-lib/commitment" + "github.com/ledgerwatch/erigon-lib/common" common2 "github.com/ledgerwatch/erigon-lib/common" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/cmp" @@ -44,6 +46,7 @@ import ( "github.com/ledgerwatch/erigon/consensus" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" + "github.com/ledgerwatch/erigon/core/rawdb/rawdbreset" reset2 "github.com/ledgerwatch/erigon/core/rawdb/rawdbreset" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" @@ -316,6 +319,379 @@ var cmdStageTxLookup = &cobra.Command{ } }, } + +func runResetStage(stg stages.SyncStage, mainBucket string, attrs *roaring64.Bitmap) func(cmd *cobra.Command, args []string) { + return func(cmd *cobra.Command, args []string) { + ctx := cmd.Context() + logger := debug.SetupCobra(cmd, "integration") + db, err := openDB(dbCfg(kv.ChainDB, chaindata), true, snapshotVersion, logger) + if err != nil { + logger.Error("Opening DB", "error", err) + return + } + defer db.Close() + + // Unset address attributes for associated bucket + if attrs != nil { + tx, err := db.BeginRw(ctx) + if err != nil { + logger.Error("Tx", "error", err) + return + } + defer tx.Rollback() + + bucket, err := tx.CursorDupSort(mainBucket) + if err != nil { + logger.Error("Tx", "error", err) + return + } + defer bucket.Close() + + k, v, err := bucket.First() + if err != nil { + logger.Error("Tx", "error", err) + return + } + for k != nil { + addr := common.BytesToAddress(v) + if err := stagedsync.RemoveAttributes(tx, addr, attrs); err != nil { + logger.Error("Tx", "error", err) + return + } + + k, v, err = bucket.NextDup() + if err != nil { + logger.Error("Tx", "error", err) + return + } + if k == nil { + k, v, err = bucket.NextNoDup() + if err != nil { + logger.Error("Tx", "error", err) + return + } + } + } + + if err := tx.Commit(); err != nil { + logger.Error("Tx", "error", err) + return + } + } + + // Force reset stage and associated buckets + if err := rawdbreset.Reset(ctx, db, stg); err != nil { + log.Error("Error", "err", err) + return + } + } +} + +var cmdResetOts2Alpha1 = &cobra.Command{ + Use: "reset_ots2_alpha1", + Short: "", + Run: func(cmd *cobra.Command, args []string) { + ctx := cmd.Context() + logger := debug.SetupCobra(cmd, "integration") + db, err := openDB(dbCfg(kv.ChainDB, chaindata), true, snapshotVersion, logger) + if err != nil { + logger.Error("Opening DB", "error", err) + return + } + defer db.Close() + + bucketsList := []string{ + kv.OtsAllContracts, + kv.OtsERC20, + kv.OtsERC165, + kv.OtsERC721, + kv.OtsERC1155, + kv.OtsERC1167, + kv.OtsERC4626, + + kv.OtsAllContractsCounter, + kv.OtsERC20Counter, + kv.OtsERC165Counter, + kv.OtsERC721Counter, + kv.OtsERC1155Counter, + kv.OtsERC1167Counter, + kv.OtsERC4626Counter, + + kv.OtsAddrAttributes, + + kv.OtsERC20TransferIndex, + kv.OtsERC20TransferCounter, + kv.OtsERC721TransferIndex, + kv.OtsERC721TransferCounter, + kv.OtsERC20Holdings, + kv.OtsERC721Holdings, + } + stagesList := []stages.SyncStage{ + stages.OtsContractIndexer, + stages.OtsERC20Indexer, + stages.OtsERC165Indexer, + stages.OtsERC721Indexer, + stages.OtsERC1155Indexer, + stages.OtsERC1167Indexer, + stages.OtsERC4626Indexer, + stages.OtsERC20And721Transfers, + stages.OtsERC20And721Holdings, + } + if err := db.Update(ctx, func(tx kv.RwTx) error { + for _, b := range bucketsList { + if err := tx.ClearBucket(b); err != nil { + return err + } + } + if err := rawdbreset.ClearStageProgress(tx, stagesList...); err != nil { + return err + } + return nil + }); err != nil { + log.Error("Error", "err", err) + return + } + }, +} + +var cmdResetOtsAllContracts = &cobra.Command{ + Use: "reset_ots_all_contracts", + Short: "", + Run: runResetStage(stages.OtsContractIndexer, kv.OtsAllContracts, nil), +} + +var cmdResetOtsERC20 = &cobra.Command{ + Use: "reset_ots_erc20", + Short: "", + Run: runResetStage(stages.OtsERC20Indexer, kv.OtsERC20, roaring64.BitmapOf(kv.ADDR_ATTR_ERC20)), +} + +var cmdResetOtsERC165 = &cobra.Command{ + Use: "reset_ots_erc165", + Short: "", + Run: runResetStage(stages.OtsERC165Indexer, kv.OtsERC165, roaring64.BitmapOf(kv.ADDR_ATTR_ERC165)), +} + +var cmdResetOtsERC721 = &cobra.Command{ + Use: "reset_ots_erc721", + Short: "", + Run: runResetStage(stages.OtsERC721Indexer, kv.OtsERC721, roaring64.BitmapOf(kv.ADDR_ATTR_ERC721, kv.ADDR_ATTR_ERC721_MD)), +} + +var cmdResetOtsERC1155 = &cobra.Command{ + Use: "reset_ots_erc1155", + Short: "", + Run: runResetStage(stages.OtsERC1155Indexer, kv.OtsERC1155, roaring64.BitmapOf(kv.ADDR_ATTR_ERC1155)), +} + +var cmdResetOtsERC1167 = &cobra.Command{ + Use: "reset_ots_erc1167", + Short: "", + Run: runResetStage(stages.OtsERC1167Indexer, kv.OtsERC1167, roaring64.BitmapOf(kv.ADDR_ATTR_ERC1167)), +} + +var cmdResetOtsERC4626 = &cobra.Command{ + Use: "reset_ots_erc4626", + Short: "", + Run: runResetStage(stages.OtsERC4626Indexer, kv.OtsERC4626, roaring64.BitmapOf(kv.ADDR_ATTR_ERC4626)), +} + +var cmdResetOtsERC20And721Transfers = &cobra.Command{ + Use: "reset_ots_erc20_721_transfers", + Short: "", + Run: runResetStage(stages.OtsERC20And721Transfers, "", nil), +} + +var cmdResetOtsERC20And721Holdings = &cobra.Command{ + Use: "reset_ots_erc20_721_holdings", + Short: "", + Run: runResetStage(stages.OtsERC20And721Holdings, "", nil), +} + +var cmdResetOtsBlocksRewarded = &cobra.Command{ + Use: "reset_ots_blocks_rewarded", + Short: "", + Run: runResetStage(stages.OtsBlocksRewarded, "", nil), +} + +var cmdResetOtsWithdrawals = &cobra.Command{ + Use: "reset_ots_withdrawals", + Short: "", + Run: runResetStage(stages.OtsWithdrawals, "", nil), +} + +func runUnwindStage(stg stages.SyncStage, mainBucket, counterBucket string, attrs *roaring64.Bitmap) func(cmd *cobra.Command, args []string) { + return func(cmd *cobra.Command, args []string) { + ctx := cmd.Context() + logger := debug.SetupCobra(cmd, "integration") + db, err := openDB(dbCfg(kv.ChainDB, chaindata), true, snapshotVersion, logger) + if err != nil { + logger.Error("Opening DB", "error", err) + return + } + defer db.Close() + + unwinder := stagedsync.NewGenericIndexerUnwinder( + mainBucket, + counterBucket, + attrs, + ) + otsUnwind(ctx, db, stg, unwinder, logger) + } +} + +func runUnwindLogStage(stg stages.SyncStage, unwinder stagedsync.UnwindExecutor) func(cmd *cobra.Command, args []string) { + return func(cmd *cobra.Command, args []string) { + ctx := cmd.Context() + logger := debug.SetupCobra(cmd, "integration") + db, err := openDB(dbCfg(kv.ChainDB, chaindata), true, snapshotVersion, logger) + if err != nil { + logger.Error("Opening DB", "error", err) + return + } + defer db.Close() + + otsUnwind(ctx, db, stg, unwinder, logger) + } +} + +func otsUnwind(ctx context.Context, db kv.RwDB, stg stages.SyncStage, unwinder stagedsync.UnwindExecutor, logger log.Logger) { + chainConfig := fromdb.ChainConfig(db) + dirs := datadir.New(datadirCli) + engine, _, sync, _, _ := newSync(ctx, db, nil, logger) + must(sync.SetCurrentStage(stg)) + + var batchSize datasize.ByteSize + must(batchSize.UnmarshalText([]byte(batchSizeStr))) + + s := stage(sync, nil, db, stg) + + log.Info("Stage", "name", s.ID, "progress", s.BlockNumber) + + br, _ := blocksIO(db, logger) + cfg := stagedsync.StageDbAwareCfg(db, dirs.Tmp, chainConfig, br, engine) + if unwind > 0 { + u := sync.NewUnwindState(stg, s.BlockNumber-unwind, s.BlockNumber) + err := stagedsync.GenericStageUnwindImpl(ctx, nil, cfg, u, unwinder) + if err != nil { + log.Error("Error", "err", err) + } + } +} + +var cmdUnwindOtsAllContracts = &cobra.Command{ + Use: "unwind_ots_all_contracts", + Short: "", + Run: runUnwindStage( + stages.OtsContractIndexer, + kv.OtsAllContracts, + kv.OtsAllContractsCounter, + nil, + ), +} + +var cmdUnwindOtsERC20 = &cobra.Command{ + Use: "unwind_ots_erc20", + Short: "", + Run: runUnwindStage( + stages.OtsERC20Indexer, + kv.OtsERC20, + kv.OtsERC20Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC20), + ), +} + +var cmdUnwindOtsERC165 = &cobra.Command{ + Use: "unwind_ots_erc165", + Short: "", + Run: runUnwindStage( + stages.OtsERC165Indexer, + kv.OtsERC165, + kv.OtsERC165Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC165), + ), +} + +var cmdUnwindOtsERC721 = &cobra.Command{ + Use: "unwind_ots_erc721", + Short: "", + Run: runUnwindStage( + stages.OtsERC721Indexer, + kv.OtsERC721, + kv.OtsERC721Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC721, kv.ADDR_ATTR_ERC721_MD), + ), +} + +var cmdUnwindOtsERC1155 = &cobra.Command{ + Use: "unwind_ots_erc1155", + Short: "", + Run: runUnwindStage( + stages.OtsERC1155Indexer, + kv.OtsERC1155, + kv.OtsERC1155Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC1155), + ), +} + +var cmdUnwindOtsERC1167 = &cobra.Command{ + Use: "unwind_ots_erc1167", + Short: "", + Run: runUnwindStage( + stages.OtsERC1167Indexer, + kv.OtsERC1167, + kv.OtsERC1167Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC1167), + ), +} + +var cmdUnwindOtsERC4626 = &cobra.Command{ + Use: "unwind_ots_erc4626", + Short: "", + Run: runUnwindStage( + stages.OtsERC4626Indexer, + kv.OtsERC4626, + kv.OtsERC4626Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC4626), + ), +} + +var cmdUnwindOtsERC70And721Transfers = &cobra.Command{ + Use: "unwind_ots_erc20_721_transfers", + Short: "", + Run: runUnwindLogStage( + stages.OtsERC20And721Transfers, + stagedsync.NewGenericLogIndexerUnwinder(), + ), +} + +var cmdUnwindOtsERC70And721Holdings = &cobra.Command{ + Use: "unwind_ots_erc20_721_holdings", + Short: "", + Run: runUnwindLogStage( + stages.OtsERC20And721Holdings, + stagedsync.NewGenericLogHoldingsUnwinder(), + ), +} + +var cmdUnwindOtsBlocksRewarded = &cobra.Command{ + Use: "unwind_ots_blocks_rewarded", + Short: "", + Run: runUnwindLogStage( + stages.OtsBlocksRewarded, + stagedsync.NewGenericBlockIndexerUnwinder(kv.OtsBlocksRewardedIndex, kv.OtsBlocksRewardedCounter, stagedsync.RunBlocksRewardedBlockUnwind), + ), +} + +var cmdUnwindOtsWithdrawals = &cobra.Command{ + Use: "unwind_ots_withdrawals", + Short: "", + Run: runUnwindLogStage( + stages.OtsWithdrawals, + stagedsync.NewGenericBlockIndexerUnwinder(kv.OtsWithdrawalsIndex, kv.OtsWithdrawalsCounter, stagedsync.RunWithdrawalsBlockUnwind), + ), +} + var cmdPrintStages = &cobra.Command{ Use: "print_stages", Short: "", @@ -607,6 +983,98 @@ func init() { rootCmd.AddCommand(cmdStageTxLookup) withConfig(cmdPrintMigrations) + + withDataDir(cmdResetOts2Alpha1) + rootCmd.AddCommand(cmdResetOts2Alpha1) + + withDataDir(cmdResetOtsAllContracts) + rootCmd.AddCommand(cmdResetOtsAllContracts) + + withDataDir(cmdResetOtsERC20) + rootCmd.AddCommand(cmdResetOtsERC20) + + withDataDir(cmdResetOtsERC165) + rootCmd.AddCommand(cmdResetOtsERC165) + + withDataDir(cmdResetOtsERC721) + rootCmd.AddCommand(cmdResetOtsERC721) + + withDataDir(cmdResetOtsERC1155) + rootCmd.AddCommand(cmdResetOtsERC1155) + + withDataDir(cmdResetOtsERC1167) + rootCmd.AddCommand(cmdResetOtsERC1167) + + withDataDir(cmdResetOtsERC4626) + rootCmd.AddCommand(cmdResetOtsERC4626) + + withDataDir(cmdResetOtsERC20And721Transfers) + rootCmd.AddCommand(cmdResetOtsERC20And721Transfers) + + withDataDir(cmdResetOtsERC20And721Holdings) + rootCmd.AddCommand(cmdResetOtsERC20And721Holdings) + + withDataDir(cmdResetOtsBlocksRewarded) + rootCmd.AddCommand(cmdResetOtsBlocksRewarded) + + withDataDir(cmdResetOtsWithdrawals) + rootCmd.AddCommand(cmdResetOtsWithdrawals) + + withDataDir(cmdUnwindOtsAllContracts) + withChain(cmdUnwindOtsAllContracts) + withUnwind(cmdUnwindOtsAllContracts) + rootCmd.AddCommand(cmdUnwindOtsAllContracts) + + withDataDir(cmdUnwindOtsERC20) + withChain(cmdUnwindOtsERC20) + withUnwind(cmdUnwindOtsERC20) + rootCmd.AddCommand(cmdUnwindOtsERC20) + + withDataDir(cmdUnwindOtsERC165) + withChain(cmdUnwindOtsERC165) + withUnwind(cmdUnwindOtsERC165) + rootCmd.AddCommand(cmdUnwindOtsERC165) + + withDataDir(cmdUnwindOtsERC721) + withChain(cmdUnwindOtsERC721) + withUnwind(cmdUnwindOtsERC721) + rootCmd.AddCommand(cmdUnwindOtsERC721) + + withDataDir(cmdUnwindOtsERC1155) + withChain(cmdUnwindOtsERC1155) + withUnwind(cmdUnwindOtsERC1155) + rootCmd.AddCommand(cmdUnwindOtsERC1155) + + withDataDir(cmdUnwindOtsERC1167) + withChain(cmdUnwindOtsERC1167) + withUnwind(cmdUnwindOtsERC1167) + rootCmd.AddCommand(cmdUnwindOtsERC1167) + + withDataDir(cmdUnwindOtsERC4626) + withChain(cmdUnwindOtsERC4626) + withUnwind(cmdUnwindOtsERC4626) + rootCmd.AddCommand(cmdUnwindOtsERC4626) + + withDataDir(cmdUnwindOtsERC70And721Transfers) + withChain(cmdUnwindOtsERC70And721Transfers) + withUnwind(cmdUnwindOtsERC70And721Transfers) + rootCmd.AddCommand(cmdUnwindOtsERC70And721Transfers) + + withDataDir(cmdUnwindOtsERC70And721Holdings) + withChain(cmdUnwindOtsERC70And721Holdings) + withUnwind(cmdUnwindOtsERC70And721Holdings) + rootCmd.AddCommand(cmdUnwindOtsERC70And721Holdings) + + withDataDir(cmdUnwindOtsBlocksRewarded) + withChain(cmdUnwindOtsBlocksRewarded) + withUnwind(cmdUnwindOtsBlocksRewarded) + rootCmd.AddCommand(cmdUnwindOtsBlocksRewarded) + + withDataDir(cmdUnwindOtsWithdrawals) + withChain(cmdUnwindOtsWithdrawals) + withUnwind(cmdUnwindOtsWithdrawals) + rootCmd.AddCommand(cmdUnwindOtsWithdrawals) + withDataDir(cmdPrintMigrations) withSnapshotVersion(cmdPrintMigrations) rootCmd.AddCommand(cmdPrintMigrations) @@ -1564,6 +2032,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig, cfg.Dirs = datadir.New(datadirCli) allSn, _, agg := allSnapshots(ctx, db, snapshotVersion, logger) cfg.Snapshot = allSn.Cfg() + cfg.Ots2 = true blockReader, blockWriter := blocksIO(db, logger) engine, heimdallClient := initConsensusEngine(ctx, chainConfig, cfg.Dirs.DataDir, db, blockReader, logger) diff --git a/cmd/rpcdaemon/rpcservices/eth_backend.go b/cmd/rpcdaemon/rpcservices/eth_backend.go index aa4f8192ee0..8506a9072a8 100644 --- a/cmd/rpcdaemon/rpcservices/eth_backend.go +++ b/cmd/rpcdaemon/rpcservices/eth_backend.go @@ -268,6 +268,19 @@ func (back *RemoteBackend) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, bl func (back *RemoteBackend) EventLookup(ctx context.Context, tx kv.Getter, txnHash common.Hash) (uint64, bool, error) { return back.blockReader.EventLookup(ctx, tx, txnHash) } + +func (back *RemoteBackend) TxnByTxId(ctx context.Context, tx kv.Getter, txId uint64) (types.Transaction, error) { + return back.blockReader.TxnByTxId(ctx, tx, txId) +} + +func (back *RemoteBackend) TxIdByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, i int) (uint64, error) { + return back.blockReader.TxIdByIdxInBlock(ctx, tx, blockNum, i) +} + +func (back *RemoteBackend) BaseTxIdForBlock(ctx context.Context, tx kv.Getter, blockNum uint64) (txid uint64, err error) { + return back.blockReader.BaseTxIdForBlock(ctx, tx, blockNum) +} + func (back *RemoteBackend) EventsByBlock(ctx context.Context, tx kv.Tx, hash common.Hash, blockNum uint64) ([]rlp.RawValue, error) { return back.blockReader.EventsByBlock(ctx, tx, hash, blockNum) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 23bb4fa55c8..e93c0a438ba 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -832,6 +832,11 @@ var ( Usage: "Max allowed page size for search methods", Value: 25, } + OtsV2Flag = cli.BoolFlag{ + Name: "experimental.ots2", + Usage: "Enable experimental Otterscan API V2", + Value: false, + } DiagnosticsURLFlag = cli.StringFlag{ Name: "diagnostics.addr", @@ -860,7 +865,6 @@ var ( Name: "silkworm.sentry", Usage: "Enable embedded Silkworm Sentry service", } - BeaconAPIFlag = cli.BoolFlag{ Name: "beacon.api", Usage: "Enable beacon API", @@ -1762,6 +1766,8 @@ func SetEthConfig(ctx *cli.Context, nodeConfig *nodecfg.Config, cfg *ethconfig.C if ctx.IsSet(TxPoolGossipDisableFlag.Name) { cfg.DisableTxPoolGossip = ctx.Bool(TxPoolGossipDisableFlag.Name) } + + cfg.Ots2 = ctx.Bool(OtsV2Flag.Name) } // SetDNSDiscoveryDefaults configures DNS discovery with the given URL if diff --git a/core/rawdb/rawdbreset/reset_stages.go b/core/rawdb/rawdbreset/reset_stages.go index b791e0e84cf..286eefae93c 100644 --- a/core/rawdb/rawdbreset/reset_stages.go +++ b/core/rawdb/rawdbreset/reset_stages.go @@ -186,6 +186,18 @@ var Tables = map[stages.SyncStage][]string{ stages.AccountHistoryIndex: {kv.E2AccountsHistory}, stages.StorageHistoryIndex: {kv.E2StorageHistory}, stages.Finish: {}, + + stages.OtsContractIndexer: {kv.OtsAllContracts, kv.OtsAllContractsCounter}, + stages.OtsERC20Indexer: {kv.OtsERC20, kv.OtsERC20Counter}, + stages.OtsERC165Indexer: {kv.OtsERC165, kv.OtsERC165Counter}, + stages.OtsERC721Indexer: {kv.OtsERC721, kv.OtsERC721Counter}, + stages.OtsERC1155Indexer: {kv.OtsERC1155, kv.OtsERC1155Counter}, + stages.OtsERC1167Indexer: {kv.OtsERC1167, kv.OtsERC1167Counter}, + stages.OtsERC4626Indexer: {kv.OtsERC4626, kv.OtsERC4626Counter}, + stages.OtsERC20And721Transfers: {kv.OtsERC20TransferIndex, kv.OtsERC20TransferCounter, kv.OtsERC721TransferIndex, kv.OtsERC721TransferCounter}, + stages.OtsERC20And721Holdings: {kv.OtsERC20Holdings, kv.OtsERC721Holdings}, + stages.OtsBlocksRewarded: {kv.OtsBlocksRewardedIndex, kv.OtsBlocksRewardedCounter}, + stages.OtsWithdrawals: {kv.OtsWithdrawalIdx2Block, kv.OtsWithdrawalsIndex, kv.OtsWithdrawalsCounter}, } var stateBuckets = []string{ kv.PlainState, kv.HashedAccounts, kv.HashedStorage, kv.TrieOfAccounts, kv.TrieOfStorage, @@ -216,6 +228,10 @@ var stateHistoryV4Buckets = []string{ kv.TblCommitmentKeys, kv.TblCommitmentVals, kv.TblCommitmentHistoryKeys, kv.TblCommitmentHistoryVals, kv.TblCommitmentIdx, } +func ClearStageProgress(tx kv.RwTx, stagesList ...stages.SyncStage) error { + return clearStageProgress(tx, stagesList...) +} + func clearStageProgress(tx kv.RwTx, stagesList ...stages.SyncStage) error { for _, stage := range stagesList { if err := stages.SaveStageProgress(tx, stage, 0); err != nil { diff --git a/core/state/plain_readonly.go b/core/state/plain_readonly.go index 9f1337f4e95..433c1f90983 100644 --- a/core/state/plain_readonly.go +++ b/core/state/plain_readonly.go @@ -56,6 +56,13 @@ type PlainState struct { systemContractLookup map[libcommon.Address][]libcommon.CodeRecord } +func (s *PlainState) Dispose() { + s.accHistoryC.Close() + s.storageHistoryC.Close() + s.accChangesC.Close() + s.storageChangesC.Close() +} + func NewPlainState(tx kv.Tx, blockNr uint64, systemContractLookup map[libcommon.Address][]libcommon.CodeRecord) *PlainState { histV3, _ := kvcfg.HistoryV3.Enabled(tx) if histV3 { diff --git a/docs/otterscan-indexers.md b/docs/otterscan-indexers.md new file mode 100644 index 00000000000..086607268db --- /dev/null +++ b/docs/otterscan-indexers.md @@ -0,0 +1,410 @@ +# Otterscan Indexers Design Document + +## Introduction + +This document describes the customizations made to support Otterscan optional stages/indexes. + +Some changes were made to `erigon-lib`, but they are explained here. + +## General conventions + +### Database + +Every Otterscan owned table is prefixed with `Ots` to keep a clear separation from Erigon core tables. + +Their declarations can be found in `erigon-lib`, in `tables.go`. + +### Custom indexers + +Every Otterscan custom stage is prefixed with `Ots`to keep it clear that it is a custom stage added by Otterscan. + +Their declarations can be found in `eth/stagedsync/stages/stages.go`. + +## Contract classifiers + +### Index tables + +Let's first define what is an unique index. For example, all contracts that adhere to the ERC20 standard can be considered an unique index. + +In fact, most built-in indexes are attempts to classify deployed contracts into different ERC standards. + +For each unique index, there will be a pair of tables `Ots`/`OtsCounter`. + +The `Ots` table is a DupSorted table with the following standard structure: + +| Key | Value | +| --------------------- | ---------------------------------------- | +| `uint64` block number | `[20]byte` contract address + extra data | + +That means: the contract at `address` was deployed at `block number` and complies to whatever the indexer that produced this insertion does. + +Using `block number` as key provide us a reorg-friendly temporal data structure to undo insertions. + +The table pair must be declared in `tables.go` at `erigon-lib` in order to be available for use. There must be a `const` name declaration + whitelisting the table in the `ChaindataTables` array. The non-`Count` table must also be declared as DupSort in `ChaindataTablesCfg` map. + +The `OtsCounter` table is a regular table used for results pagination and it has the following structure: + +| Key | Value | +| ---------------------------- | --------------------- | +| `uint64` accumulated counter | `uint64` block number | + +That means: up to `block number` (inclusive), there are `counter` search results. + +So if one wants to find the Nth result inside the index (1-based), you must turn `N` into the key and search for it. It'll find the block number the contains the result. + +Then get the key of previous record (previous accumulated count), calculate `N` - previous count + 1. The result (0-based) will let you know which match position inside the block corresponds to the desired match. + +## Index stages + +Each unique index is defined by a custom stage in `eth/stagedsync/ots_indexer_.go` and a standardized name in `eth/stagedsync/stages/stages.go`. + +The custom stage is added to the Erigon stage loop by adding injecting its declaration on `eth/stagedsync/default_stages.go`. + +The stages use a custom stage framework so most of common code is abstracted away. More on that later. + +Suffice to say that each of these stages operate on an unique index, filtering data from a set of `Ots/OtsCounter` tables to its own set. The upper bound block number is also defined by its "parent" stage. + +For example, the ERC721 classifier index stage is bound to the ERC165 classifier stage, since every ERC721 must implement ERC165, so we reduce significantly the universe of contracts to be analyzed by making it a child of ERC165 stage. + +As an utility "hack", a "reset" command to cleanup tables should be created in `cmd/integration/commands/stages.go` along with reset instructions in `core/rawdb/rawdbreset/reset_stages.go`, `Tables` mapping. + +## Benckmarks + +Reference hardware: gcloud n2-standard-8 + +### Total disk space (27/08/2023) + +Baseline mainnet: + +wmitsuda@mainnet-wmitsuda-2:/mnt/erigon$ du -hd1 . +410G ./snapshots +16K ./lost+found +188M ./txpool +195M ./logs +21M ./nodes +1.4M ./temp +1.9T ./chaindata +2.3T . + +After: + +wmitsuda@mainnet-wmitsuda-2:/mnt/erigon$ du -hd1 . +412G ./snapshots +16K ./lost+found +291M ./txpool +211M ./logs +22M ./nodes +1.6M ./temp +1.9T ./chaindata +2.3T . + +### All contracts (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|09:22:02.856] [15/24 OtsContractIndexer] Finished latest=18080701 +[INFO] [09-07|09:22:05.706] [15/24 OtsContractIndexer] DONE in=1h57m7.535823984s +``` + +Space: (2.7GB + 190MB == 2.9GB) + +``` +Status of OtsAllContracts + Pagesize: 4096 + Tree depth: 4 + Branch pages: 79322 + Leaf pages: 585383 + Overflow pages: 0 + Entries: 57806584 +Status of OtsAllContractsCounter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 206 + Leaf pages: 46120 + Overflow pages: 0 + Entries: 7194585 +``` + +### ERC20 (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|10:32:21.222] [16/24 OtsERC20Indexer] Totals totalMatches=898829 totalProbed=57785855 +[INFO] [09-07|10:32:21.222] [16/24 OtsERC20Indexer] Finished latest=18080701 +[INFO] [09-07|10:32:21.584] [16/24 OtsERC20Indexer] DONE in=1h10m15.877153125s +``` + +Space: (48MB + 19MB == 67MB) + +``` +Status of OtsERC20 + Pagesize: 4096 + Tree depth: 3 + Branch pages: 58 + Leaf pages: 11807 + Overflow pages: 0 + Entries: 899283 +Status of OtsERC20Counter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 22 + Leaf pages: 4687 + Overflow pages: 0 + Entries: 731079 +``` + +### ERC165 (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|11:42:16.335] [17/24 OtsERC165Indexer] Totals totalMatches=3806792 totalProbed=57785855 +[INFO] [09-07|11:42:16.335] [17/24 OtsERC165Indexer] Finished latest=18080701 +[INFO] [09-07|11:42:17.734] [17/24 OtsERC165Indexer] DONE in=1h9m56.149619591s +``` + +Space: (150MB + 17MB == 167MB) + +``` +Status of OtsERC165 + Pagesize: 4096 + Tree depth: 3 + Branch pages: 345 + Leaf pages: 36187 + Overflow pages: 0 + Entries: 3809924 +Status of OtsERC165Counter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 20 + Leaf pages: 4167 + Overflow pages: 0 + Entries: 650032 +``` + +### ERC721 (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|11:49:04.381] [18/24 OtsERC721Indexer] Totals totalMatches=3314490 totalProbed=3806792 +[INFO] [09-07|11:49:04.381] [18/24 OtsERC721Indexer] Finished latest=18080701 +[INFO] [09-07|11:49:05.778] [18/24 OtsERC721Indexer] DONE in=6m48.043524943s +``` + +Space: (128MB + 8MB == 136MB) + +``` +Status of OtsERC721 + Pagesize: 4096 + Tree depth: 3 + Branch pages: 274 + Leaf pages: 30897 + Overflow pages: 0 + Entries: 3317317 +Status of OtsERC721Counter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 10 + Leaf pages: 1939 + Overflow pages: 0 + Entries: 302365 +``` + +### ERC1155 (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|11:54:51.061] [19/24 OtsERC1155Indexer] Totals totalMatches=49185 totalProbed=3806792 +[INFO] [09-07|11:54:51.061] [19/24 OtsERC1155Indexer] Finished latest=18080701 +[INFO] [09-07|11:54:51.649] [19/24 OtsERC1155Indexer] DONE in=5m45.871004363s +``` + +Space: (2.1MB + 1MB == 3.1MB) + +``` +Status of OtsERC1155 + Pagesize: 4096 + Tree depth: 3 + Branch pages: 4 + Leaf pages: 522 + Overflow pages: 0 + Entries: 49198 +Status of OtsERC1155Counter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 3 + Leaf pages: 257 + Overflow pages: 0 + Entries: 39954 +``` + +### ERC1167 (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|12:42:05.034] [20/24 OtsERC1167Indexer] Totals totalMatches=13690895 totalProbed=57785855 +[INFO] [09-07|12:42:05.034] [20/24 OtsERC1167Indexer] Finished latest=18080701 +[INFO] [09-07|12:42:08.327] [20/24 OtsERC1167Indexer] DONE in=47m16.678467963s +``` + +Space: (598MB + 46MB == 644MB) + +``` +Status of OtsERC1167 + Pagesize: 4096 + Tree depth: 4 + Branch pages: 3706 + Leaf pages: 142144 + Overflow pages: 0 + Entries: 13709968 +Status of OtsERC1167Counter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 51 + Leaf pages: 11222 + Overflow pages: 0 + Entries: 1750617 +``` + +### ERC4626 (08/09/2023) + +Time (15 workers): + +``` +[INFO] [09-07|12:44:33.584] [21/24 OtsERC4626Indexer] Totals totalMatches=1478 totalProbed=898829 +[INFO] [09-07|12:44:33.584] [21/24 OtsERC4626Indexer] Finished latest=18080701 +[INFO] [09-07|12:44:33.616] [21/24 OtsERC4626Indexer] DONE in=2m25.288537184s +``` + +Space: (82KB + 37KB == 119KB) + +``` +Status of OtsERC4626 + Pagesize: 4096 + Tree depth: 2 + Branch pages: 1 + Leaf pages: 18 + Overflow pages: 0 + Entries: 1479 +Status of OtsERC4626Counter + Pagesize: 4096 + Tree depth: 2 + Branch pages: 1 + Leaf pages: 9 + Overflow pages: 0 + Entries: 1256 +``` + +### ERC20+721 Transfers (08/09/2023) + +Time: + +``` +[INFO] [09-07|23:12:03.800] [22/24 OtsERC20And721Transfers] Totals matches=852510711 txCount=1154620749 +[INFO] [09-07|23:12:03.803] [22/24 OtsERC20And721Transfers] Finished latest=18080701 +[INFO] [09-07|23:12:07.749] [22/24 OtsERC20And721Transfers] DONE in=10h27m34.132621965s +``` + +Space: (5.7GB + 23GB + 332MB + 2.3GB == 31.33GB) + +``` +Status of OtsERC20TransferCounter + Pagesize: 4096 + Tree depth: 4 + Branch pages: 12936 + Leaf pages: 1388737 + Overflow pages: 0 + Entries: 137270901 +Status of OtsERC20TransferIndex + Pagesize: 4096 + Tree depth: 5 + Branch pages: 82323 + Leaf pages: 5592031 + Overflow pages: 12994 + Entries: 137270901 +Status of OtsERC721TransferCounter + Pagesize: 4096 + Tree depth: 4 + Branch pages: 649 + Leaf pages: 80444 + Overflow pages: 0 + Entries: 8201734 +Status of OtsERC721TransferIndex + Pagesize: 4096 + Tree depth: 4 + Branch pages: 6654 + Leaf pages: 551828 + Overflow pages: 2676 + Entries: 8201734 +``` + +### ERC20+721 Holders (08/09/2023) + +Time: + +``` +[INFO] [09-08|07:36:38.849] [23/24 OtsERC20And721Holdings] Totals matches=852510711 txCount=1154620749 +[INFO] [09-08|07:36:38.850] [23/24 OtsERC20And721Holdings] Finished latest=18080701 +[INFO] [09-08|07:36:42.616] [23/24 OtsERC20And721Holdings] DONE in=8h24m34.86630761s +``` + +Space: (17GB + 3GB == 20GB) + +``` +Status of OtsERC20Holdings + Pagesize: 4096 + Tree depth: 5 + Branch pages: 148710 + Leaf pages: 4015927 + Overflow pages: 0 + Entries: 293188023 + +Status of OtsERC721Holdings + Pagesize: 4096 + Tree depth: 4 + Branch pages: 73097 + Leaf pages: 681735 + Overflow pages: 0 + Entries: 53484559 +``` + +### Withdrawals (26/09/2023) + +Time: + +``` +[INFO] [09-27|00:37:38.379] [21/22 OtsWithdrawals] Totals matches=1188617 blocks=18223557 +[INFO] [09-27|00:37:38.380] [21/22 OtsWithdrawals] Finished latest=18223556 +[INFO] [09-27|00:37:38.821] [21/22 OtsWithdrawals] DONE in=2m0.81539878s +``` + +Space (31.5MB + 4.3MB + 168.3MB == 204.1MB) + +``` +Status of OtsWithdrawalIdx2Block + Pagesize: 4096 + Tree depth: 3 + Branch pages: 36 + Leaf pages: 7655 + Overflow pages: 0 + Entries: 1194115 +Status of OtsWithdrawalsCounter + Pagesize: 4096 + Tree depth: 3 + Branch pages: 46 + Leaf pages: 1017 + Overflow pages: 0 + Entries: 126758 +Status of OtsWithdrawalsIndex + Pagesize: 4096 + Tree depth: 4 + Branch pages: 505 + Leaf pages: 40587 + Overflow pages: 34 + Entries: 126758 +``` diff --git a/erigon-lib/common/length/length.go b/erigon-lib/common/length/length.go index 09b126c769e..c5071c822e8 100644 --- a/erigon-lib/common/length/length.go +++ b/erigon-lib/common/length/length.go @@ -37,4 +37,8 @@ const ( Ts = 8 // Incarnation length of uint64 for contract incarnations Incarnation = 8 + // Chunk size used in indexes + Chunk = 8 + // Counter size used in paged indexes + Counter = 8 ) diff --git a/erigon-lib/kv/tables.go b/erigon-lib/kv/tables.go index 75435eef207..c96796c13e3 100644 --- a/erigon-lib/kv/tables.go +++ b/erigon-lib/kv/tables.go @@ -494,6 +494,37 @@ const ( Proposers = "BlockProposers" // epoch => proposers indicies StatesProcessingProgress = "StatesProcessingProgress" + + OtsAllContracts = "OtsAllContracts" + OtsERC20 = "OtsERC20" + OtsERC165 = "OtsERC165" + OtsERC721 = "OtsERC721" + OtsERC1155 = "OtsERC1155" + OtsERC1167 = "OtsERC1167" + OtsERC4626 = "OtsERC4626" + + OtsAllContractsCounter = "OtsAllContractsCounter" + OtsERC20Counter = "OtsERC20Counter" + OtsERC165Counter = "OtsERC165Counter" + OtsERC721Counter = "OtsERC721Counter" + OtsERC1155Counter = "OtsERC1155Counter" + OtsERC1167Counter = "OtsERC1167Counter" + OtsERC4626Counter = "OtsERC4626Counter" + + OtsAddrAttributes = "OtsAddrAttributes" + + OtsERC20TransferIndex = "OtsERC20TransferIndex" + OtsERC20TransferCounter = "OtsERC20TransferCounter" + OtsERC721TransferIndex = "OtsERC721TransferIndex" + OtsERC721TransferCounter = "OtsERC721TransferCounter" + OtsERC20Holdings = "OtsERC20Holdings" + OtsERC721Holdings = "OtsERC721Holdings" + + OtsBlocksRewardedIndex = "OtsBlocksRewardedIndex" + OtsBlocksRewardedCounter = "OtsBlocksRewardedCounter" + OtsWithdrawalIdx2Block = "OtsWithdrawalIdx2Block" + OtsWithdrawalsIndex = "OtsWithdrawalsIndex" + OtsWithdrawalsCounter = "OtsWithdrawalsCounter" ) // Keys @@ -683,8 +714,49 @@ var ChaindataTables = []string{ Eth1DataVotes, IntraRandaoMixes, ActiveValidatorIndicies, + + OtsAllContracts, + OtsERC20, + OtsERC165, + OtsERC721, + OtsERC1155, + OtsERC1167, + OtsERC4626, + + OtsAllContractsCounter, + OtsERC20Counter, + OtsERC165Counter, + OtsERC721Counter, + OtsERC1155Counter, + OtsERC1167Counter, + OtsERC4626Counter, + + OtsAddrAttributes, + + OtsERC20TransferIndex, + OtsERC20TransferCounter, + OtsERC721TransferIndex, + OtsERC721TransferCounter, + OtsERC20Holdings, + OtsERC721Holdings, + + OtsBlocksRewardedIndex, + OtsBlocksRewardedCounter, + OtsWithdrawalIdx2Block, + OtsWithdrawalsIndex, + OtsWithdrawalsCounter, } +const ( + ADDR_ATTR_ERC20 = 0 // is ERC20 token + ADDR_ATTR_ERC165 = 1 // implements ERC165 + ADDR_ATTR_ERC721 = 2 // implements ERC721 + ADDR_ATTR_ERC721_MD = 3 // implements ERC721 metadata + ADDR_ATTR_ERC1155 = 4 // implements ERC1155 + ADDR_ATTR_ERC1167 = 5 // is ERC1167 minimal proxy + ADDR_ATTR_ERC4626 = 6 // implements ERC4626 +) + const ( RecentLocalTransaction = "RecentLocalTransaction" // sequence_u64 -> tx_hash PoolTransaction = "PoolTransaction" // txHash -> sender_id_u64+tx_rlp @@ -795,6 +867,22 @@ var ChaindataTablesCfg = TableCfg{ RStorageIdx: {Flags: DupSort}, RCodeKeys: {Flags: DupSort}, RCodeIdx: {Flags: DupSort}, + + OtsAllContracts: {Flags: DupSort}, + OtsERC20: {Flags: DupSort}, + OtsERC165: {Flags: DupSort}, + OtsERC721: {Flags: DupSort}, + OtsERC1155: {Flags: DupSort}, + OtsERC1167: {Flags: DupSort}, + OtsERC4626: {Flags: DupSort}, + + OtsERC20TransferCounter: {Flags: DupSort}, + OtsERC721TransferCounter: {Flags: DupSort}, + OtsERC20Holdings: {Flags: DupSort}, + OtsERC721Holdings: {Flags: DupSort}, + + OtsBlocksRewardedCounter: {Flags: DupSort}, + OtsWithdrawalsCounter: {Flags: DupSort}, } var BorTablesCfg = TableCfg{ diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index a66e54cf44f..c0e76c7e5af 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -256,6 +256,8 @@ type Config struct { SilkwormSentry bool DisableTxPoolGossip bool + + Ots2 bool } type Sync struct { diff --git a/eth/stagedsync/default_stages.go b/eth/stagedsync/default_stages.go index ca5570ba797..eca494b9f23 100644 --- a/eth/stagedsync/default_stages.go +++ b/eth/stagedsync/default_stages.go @@ -27,8 +27,10 @@ func DefaultStages(ctx context.Context, callTraces CallTracesCfg, txLookup TxLookupCfg, finish FinishCfg, + caCfg ContractAnalyzerCfg, + ots2Enabled bool, test bool) []*Stage { - return []*Stage{ + defaultStages := []*Stage{ { ID: stages.Snapshots, Description: "Download snapshots", @@ -251,10 +253,22 @@ func DefaultStages(ctx context.Context, }, }, } + + // If ots2 is enabled, inject ots2 stages before finish stage + if ots2Enabled { + ots2Stages := OtsStages(ctx, caCfg) + + newStages := defaultStages[:len(defaultStages)-1] + newStages = append(newStages, ots2Stages...) + newStages = append(newStages, defaultStages[len(defaultStages)-1]) + defaultStages = newStages + } + + return defaultStages } -func PipelineStages(ctx context.Context, snapshots SnapshotsCfg, blockHashCfg BlockHashesCfg, senders SendersCfg, exec ExecuteBlockCfg, hashState HashStateCfg, trieCfg TrieCfg, history HistoryCfg, logIndex LogIndexCfg, callTraces CallTracesCfg, txLookup TxLookupCfg, finish FinishCfg, test bool) []*Stage { - return []*Stage{ +func PipelineStages(ctx context.Context, snapshots SnapshotsCfg, blockHashCfg BlockHashesCfg, senders SendersCfg, exec ExecuteBlockCfg, hashState HashStateCfg, trieCfg TrieCfg, history HistoryCfg, logIndex LogIndexCfg, callTraces CallTracesCfg, txLookup TxLookupCfg, finish FinishCfg, caCfg ContractAnalyzerCfg, ots2Enabled bool, test bool) []*Stage { + defaultStages := []*Stage{ { ID: stages.Snapshots, Description: "Download snapshots", @@ -430,6 +444,18 @@ func PipelineStages(ctx context.Context, snapshots SnapshotsCfg, blockHashCfg Bl }, }, } + + // If ots2 is enabled, inject ots2 stages before finish stage + if ots2Enabled { + ots2Stages := OtsStages(ctx, caCfg) + + newStages := defaultStages[:len(defaultStages)-1] + newStages = append(newStages, ots2Stages...) + newStages = append(newStages, defaultStages[len(defaultStages)-1]) + defaultStages = newStages + } + + return defaultStages } // when uploading - potentially from zero we need to include headers and bodies stages otherwise we won't recover the POW portion of the chain diff --git a/eth/stagedsync/ots2-dbformat.md b/eth/stagedsync/ots2-dbformat.md new file mode 100644 index 00000000000..b0ef982ea2c --- /dev/null +++ b/eth/stagedsync/ots2-dbformat.md @@ -0,0 +1,92 @@ +# Otterscan V2 DB Format + +## Match tables + +Match tables are dupsorted tables and have the following format: + +| | | | +|-----|---------------------|--------------| +| `k` | `uint64` | block number | +| `v` | `[length.Addr]byte` | address | +| | `uint64` | incarnation (if > 0) | + +This format enables some properties: + +### Determine how many matches are in a given block + +Given block number N, `var blockNum uint64 = `, `var key [length.BlockNum]byte = ` do: `k, v, err := cursor.SeekExact(key)` + +If `k` is `nil`, it means the block was not found, hence no matches in that block. + +If `k` is not `nil`, `cursor.CountDuplicates()` will give you the number of matches. + +## Counter tables + +Counter tables are non-dupsorted tables and have the following format: + +| | | | +|-----|----------|--------------------| +| `k` | `uint64` | cumulative counter | +| `v` | `uint64` | block number | + +Which reads as: for each record, there are `k` matches up to (inclusive) block number `v`. + +This format enables some properties: + +TODO: describe what can be done with this format regarding pagination. + +## Address attributes table + +This non-dupsorted table contains a plain state of `address` -> bitmap of attributes. + +Attributes are arbitrary tags defined by classifier implementations. E.g.: an address can be marked as an ERC20 token, or ERC721 NFT. + +## Transfer Index tables + +They contain indexes of address appearances under a certain context. One example of such context is the address being a from/to of an ERC20 transfer. + +Transfer index tables are non-dupsorted and have the following format: + +| | | | +|-----|---------------------|---------| +| `k` | `[length.Addr]byte` | address | +| | `uint64` | chunk ID: 0xffffffffffffffff means last chunk | +| `v` | `uint64` * N | block numbers; divide them by 8 bytes to determine how many blocks are present in the index | + +## Transfer Count tables + +They are dupsorted table and contain pagination support metadata for the transfer index tables. + +| | | | +|-----|---------------------|---------| +| `k` | `[length.Addr]byte` | address | +| `v` | `uint64` | cumulative count of matches so far | +| | `uint64` | chunk ID: 0xffffffffffffffff means last chunk | + +### Special optimization case 1 + +If the total count for a certain address is <= 256 and it fits in just 1 chunk, save it as: + +| | | | +|-----|---------------------|---------| +| `k` | `[length.Addr]byte` | address | +| `v` | `uint8` | (total count of matches) - 1 | + +Consumer of that table must certify that's the case for a certain address by checking: + +1. `cursor.CountDuplicates() == 1` +2. `len(v) == 1` + +The goal of this optimization is to optimize for the most common case which is: "address is used for testing or is involved in just a few transactions" by storing just 1 byte instead of 16 in `v`. + +## Holder Index tables + +They contain records of first token appearances for a certain address. + +Transfer index tables are dupsorted and have the following format: + +| | | | +|-----|---------------------|----------------| +| `k` | `[length.Addr]byte` | holder address | +| `v` | `[length.Addr]byte` | token address | +| | `uint64` | ethTx of first appearance (i.e. first tx from genesis that allowed us to identify that address as a holder of that token) | diff --git a/eth/stagedsync/ots_address_prober.go b/eth/stagedsync/ots_address_prober.go new file mode 100644 index 00000000000..cc9ae0f4d2a --- /dev/null +++ b/eth/stagedsync/ots_address_prober.go @@ -0,0 +1,146 @@ +package stagedsync + +import ( + "context" + "fmt" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" +) + +// A Prober evaluates the contents of a certain address and determines if it passes or fails the test +// it is programmed to. +// +// What/how it tests is determined by the implementation and is out of scope of this interface. +// We only care about yes/no/error. +// +// Users of this interface can use the result of the test to decide this address belongs to a certain +// class of content, i.e., someone could use an ERC20 prober implementation to detect ERC20 token +// contracts and index them in the DB. +type Prober interface { + // Given an EVM context passed as parameters to this function, return if it passes the test or it errored. + // + // addr is the ETH address which is being analyzed. Contract probers will usually target this address + // and staticcall functions to determine if it complies to what is being probed. + // + // sourceK/sourceV are the original key/value from the source bucket. Probers will usually ignore + // them unless it is an advanced prober that is aware of the source bucket and wants to do a in-depth + // check. + // + // It the returned attrs bitmap is nil, it means the address does not match any of the characteristics + // this prober implementation is programmed to analyze. Otherwise it returns a bitmap of attributes. + Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, sourceK, sourceV []byte) (attrs *roaring64.Bitmap, err error) +} + +// Creates a Prober instance +type ProberFactory func() (Prober, error) + +// TODO: remove and rename the "2" variant +func probeContractWithArgs(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, addr common.Address, abi *abi.ABI, data *[]byte, outputName string) ([]interface{}, error) { + // Use block gas limit for the call + gas := hexutil.Uint64(header.GasLimit) + args := ethapi.CallArgs{ + To: &addr, + Data: (*hexutility.Bytes)(data), + Gas: &gas, + } + + ret, err := probeContract(ctx, evm, header, chainConfig, ibs, args) + if err != nil { + return nil, err + } + if ret.Err != nil { + // ignore errors because we are probing untrusted contract + return nil, nil + } + res, err := abi.Unpack(outputName, ret.ReturnData) + if err != nil { + // ignore errors because we are probing untrusted contract + return nil, nil + } + + return res, nil +} + +func probeContractWithArgs2(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, addr common.Address, abi *abi.ABI, data *[]byte, outputName string) ([]interface{}, error, *core.ExecutionResult) { + // Use block gas limit for the call + gas := hexutil.Uint64(header.GasLimit) + args := ethapi.CallArgs{ + To: &addr, + Data: (*hexutility.Bytes)(data), + Gas: &gas, + } + + ret, err := probeContract(ctx, evm, header, chainConfig, ibs, args) + if err != nil { + return nil, err, nil + } + if ret.Failed() { + // ignore errors because we are probing untrusted contract + return nil, nil, nil + } + res, err := abi.Unpack(outputName, ret.ReturnData) + if err != nil { + // ignore errors because we are probing untrusted contract + return nil, nil, nil + } + + return res, nil, ret +} + +func expectRevert(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, addr *common.Address, data *[]byte) (bool, error, *core.ExecutionResult) { + gas := hexutil.Uint64(header.GasLimit) + args := ethapi.CallArgs{ + To: addr, + Data: (*hexutility.Bytes)(data), + Gas: &gas, + } + ret, err := probeContract(ctx, evm, header, chainConfig, ibs, args) + if err != nil { + // internal error + return false, err, nil + } + + return ret.Failed(), nil, ret +} + +func probeContract(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, state *state.IntraBlockState, args ethapi.CallArgs) (*core.ExecutionResult, error) { + var baseFee *uint256.Int + if header != nil && header.BaseFee != nil { + var overflow bool + baseFee, overflow = uint256.FromBig(header.BaseFee) + if overflow { + return nil, fmt.Errorf("header.BaseFee uint256 overflow") + } + } + msg, err := args.ToMessage(0, baseFee) + if err != nil { + return nil, err + } + + txCtx := core.NewEVMTxContext(msg) + state.Reset() + evm.Reset(txCtx, state) + + gp := new(core.GasPool).AddGas(msg.Gas()) + result, err := core.ApplyMessage(evm, msg, gp, true /* refunds */, false /* gasBailout */) + if err != nil { + return nil, err + } + + // If the timer caused an abort, return an appropriate error message + if evm.Cancelled() { + return nil, fmt.Errorf("execution aborted (timeout = )") + } + return result, nil +} diff --git a/eth/stagedsync/ots_index_handler.go b/eth/stagedsync/ots_index_handler.go new file mode 100644 index 00000000000..0d923ea28e4 --- /dev/null +++ b/eth/stagedsync/ots_index_handler.go @@ -0,0 +1,20 @@ +package stagedsync + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type ResourceAwareIndexHandler interface { + Flush(force bool) error + Load(ctx context.Context, tx kv.RwTx) error + Close() +} + +// An IndexHandler handle a session of addresses indexing +type IndexHandler interface { + ResourceAwareIndexHandler + TouchIndex(addr common.Address, idx uint64) +} diff --git a/eth/stagedsync/ots_index_handler_standard.go b/eth/stagedsync/ots_index_handler_standard.go new file mode 100644 index 00000000000..da7b6470506 --- /dev/null +++ b/eth/stagedsync/ots_index_handler_standard.go @@ -0,0 +1,244 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/ots/indexer" +) + +// Standard (in a meaning everyone is meant to use it) implementation of +// IndexHandler +type StandardIndexHandler struct { + indexBucket string + counterBucket string + collector *etl.Collector + bitmaps map[string]*roaring64.Bitmap +} + +func (h *StandardIndexHandler) TouchIndex(addr common.Address, idx uint64) { + bm, ok := h.bitmaps[string(addr.Bytes())] + if !ok { + bm = roaring64.NewBitmap() + h.bitmaps[string(addr.Bytes())] = bm + } + bm.Add(idx) +} + +func (h *StandardIndexHandler) Flush(force bool) error { + if force || needFlush64(h.bitmaps, bitmapsBufLimit) { + if err := flushBitmaps64(h.collector, h.bitmaps); err != nil { + return err + } + h.bitmaps = map[string]*roaring64.Bitmap{} + } + + return nil +} + +func (h *StandardIndexHandler) Load(ctx context.Context, tx kv.RwTx) error { + transferCounter, err := tx.RwCursorDupSort(h.counterBucket) + if err != nil { + return err + } + defer transferCounter.Close() + + buf := bytes.NewBuffer(nil) + addrBm := roaring64.NewBitmap() + + loadFunc := func(k []byte, value []byte, tableReader etl.CurrentTableReader, next etl.LoadNextFunc) error { + // Bitmap for address key + if _, err := addrBm.ReadFrom(bytes.NewBuffer(value)); err != nil { + return err + } + + // Last chunk for address key + addr := k[:length.Addr] + + // Read last chunk from DB (may not exist) + // Chunk already exists; merge it + if err := mergeLastChunk(addrBm, addr, tableReader); err != nil { + return err + } + + // Recover and delete the last counter (may not exist); will be replaced after this chunk write + prevCounter := uint64(0) + isUniqueChunk := false + counterK, _, err := transferCounter.SeekExact(addr) + if err != nil { + return err + } + if counterK != nil { + counterV, err := transferCounter.LastDup() + if err != nil { + return err + } + if len(counterV) == 1 { + // Optimized counter; prevCounter must remain 0 + c, err := transferCounter.CountDuplicates() + if err != nil { + return err + } + if c != 1 { + return fmt.Errorf("db possibly corrupted: bucket=%s addr=%s has optimized counter with duplicates", h.counterBucket, hexutility.Encode(addr)) + } + + isUniqueChunk = true + } else { + // Regular counter + chunk := counterV[8:] + chunkAsNumber := binary.BigEndian.Uint64(chunk) + if chunkAsNumber != ^uint64(0) { + return fmt.Errorf("db possibly corrupted: bucket=%s addr=%s last chunk is not 0xffffffffffffffff: %s", h.counterBucket, hexutility.Encode(addr), hexutility.Encode(chunk)) + } + } + + // Delete last counter, optimized or not; it doesn't matter, it'll be + // rewriten below + if err := transferCounter.DeleteCurrent(); err != nil { + return err + } + + // Regular chunk, rewind to previous counter + if !isUniqueChunk { + prevK, prevV, err := transferCounter.PrevDup() + if err != nil { + return err + } + if prevK != nil { + prevCounter = binary.BigEndian.Uint64(prevV[:8]) + } + } + } + + // Write the index chunk; cut it if necessary to fit under page restrictions + if (counterK == nil || isUniqueChunk) && prevCounter+addrBm.GetCardinality() <= 256 { + buf.Reset() + b := make([]byte, 8) + for it := addrBm.Iterator(); it.HasNext(); { + ethTx := it.Next() + binary.BigEndian.PutUint64(b, ethTx) + buf.Write(b) + } + + _, err := h.writeOptimizedChunkAndCounter(tx, k, buf, addr, next, prevCounter) + if err != nil { + return err + } + } else { + buf.Reset() + b := make([]byte, 8) + for it := addrBm.Iterator(); it.HasNext(); { + ethTx := it.Next() + binary.BigEndian.PutUint64(b, ethTx) + buf.Write(b) + + // cut? + if !it.HasNext() || buf.Len() >= int(bitmapdb.ChunkLimit) { + updatedCounter, err := h.writeRegularChunkAndCounter(tx, k, buf, addr, next, ethTx, !it.HasNext(), prevCounter) + if err != nil { + return err + } + prevCounter = updatedCounter + + // Cleanup buffer for next chunk + buf.Reset() + } + } + } + + return nil + } + if err := h.collector.Load(tx, h.indexBucket, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { + return err + } + + return nil +} + +func (h *StandardIndexHandler) writeOptimizedChunkAndCounter(tx kv.RwTx, k []byte, buf *bytes.Buffer, addr []byte, next etl.LoadNextFunc, prevCounter uint64) (uint64, error) { + // Write solo chunk + chunkKey := chunkKey(k, true, 0) + if err := next(k, chunkKey, buf.Bytes()); err != nil { + return 0, err + } + + // Write optimized counter + prevCounter += uint64(buf.Len()) / 8 + v := indexer.OptimizedCounterSerializer(prevCounter) + if err := tx.Put(h.counterBucket, addr, v); err != nil { + return 0, err + } + + return prevCounter, nil +} + +func (h *StandardIndexHandler) writeRegularChunkAndCounter(tx kv.RwTx, k []byte, buf *bytes.Buffer, addr []byte, next etl.LoadNextFunc, ethTx uint64, isLast bool, prevCounter uint64) (uint64, error) { + chunkKey := chunkKey(k, isLast, ethTx) + if err := next(k, chunkKey, buf.Bytes()); err != nil { + return 0, err + } + + // Write updated counter + prevCounter += uint64(buf.Len()) / 8 + v := indexer.RegularCounterSerializer(prevCounter, chunkKey[length.Addr:]) + if err := tx.Put(h.counterBucket, addr, v); err != nil { + return 0, err + } + + return prevCounter, nil +} + +// Reads the last index chunk for a certain address and merge the result +// into the currently being processed bitmap. +func mergeLastChunk(addrBm *roaring64.Bitmap, addr []byte, tableReader etl.CurrentTableReader) error { + chunkBm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(chunkBm) + + key := make([]byte, length.Addr+8) + copy(key, addr) + binary.BigEndian.PutUint64(key[length.Addr:], ^uint64(0)) + + // Read last chunk from DB (may not exist) + v, err := tableReader.Get(key) + if err != nil { + return err + } + if v == nil { + return nil + } + + for i := 0; i < len(v); i += 8 { + chunkBm.Add(binary.BigEndian.Uint64(v[i : i+8])) + } + addrBm.Or(chunkBm) + + return nil +} + +// k == address [length.Addr]byte + chunk uint64 +func chunkKey(k []byte, isLast bool, ethTx uint64) []byte { + key := make([]byte, length.Addr+8) + copy(key, k[:length.Addr]) + + if isLast { + binary.BigEndian.PutUint64(key[length.Addr:], ^uint64(0)) + } else { + binary.BigEndian.PutUint64(key[length.Addr:], ethTx) + } + + return key +} + +func (h *StandardIndexHandler) Close() { + h.collector.Close() +} diff --git a/eth/stagedsync/ots_index_unwinder.go b/eth/stagedsync/ots_index_unwinder.go new file mode 100644 index 00000000000..ce4640a5b03 --- /dev/null +++ b/eth/stagedsync/ots_index_unwinder.go @@ -0,0 +1,11 @@ +package stagedsync + +import ( + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type IndexUnwinder interface { + UnwindAddress(tx kv.RwTx, addr common.Address, idx uint64) error + Dispose() error +} diff --git a/eth/stagedsync/ots_indexer_body.go b/eth/stagedsync/ots_indexer_body.go new file mode 100644 index 00000000000..202be894b22 --- /dev/null +++ b/eth/stagedsync/ots_indexer_body.go @@ -0,0 +1,75 @@ +package stagedsync + +import ( + "context" + "fmt" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +type BodyIndexerHandler interface { + ResourceAwareIndexHandler + HandleMatch(blockNum uint64, body *types.Body) error +} + +// TODO: extract common logic from runIncrementalLogIndexerExecutor +func runIncrementalBodyIndexerExecutor(db kv.RoDB, tx kv.RwTx, blockReader services.FullBlockReader, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, ctx context.Context, s *StageState, handler BodyIndexerHandler) (uint64, error) { + // Tracks how many blocks finished analysis so far + totalBlocks := uint64(0) + + // Tracks how many blocks finished analysis with a match so far + totalMatch := uint64(0) + + // Process control + flushEvery := time.NewTicker(bitmapsFlushEvery) + defer flushEvery.Stop() + + // Iterate over all blocks [startBlock, endBlock] + for blockNum := startBlock; blockNum <= endBlock; blockNum++ { + hash, err := blockReader.CanonicalHash(ctx, tx, blockNum) + if err != nil { + return startBlock, err + } + body, _, err := blockReader.Body(ctx, tx, hash, blockNum) + if err != nil { + return startBlock, err + } + + totalBlocks++ + if err := handler.HandleMatch(blockNum, body); err != nil { + return startBlock, err + } + + select { + default: + case <-ctx.Done(): + return startBlock, common.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Scanning blocks", s.LogPrefix()), "block", blockNum, "matches", totalMatch, "blocks", totalBlocks) + case <-flushEvery.C: + if err := handler.Flush(false); err != nil { + return startBlock, err + } + } + } + + // Last (forced) flush and batch load (if applicable) + if err := handler.Flush(true); err != nil { + return startBlock, err + } + if err := handler.Load(ctx, tx); err != nil { + return startBlock, err + } + + // Don't print summary if no contracts were analyzed to avoid polluting logs + if !isShortInterval && totalBlocks > 0 { + log.Info(fmt.Sprintf("[%s] Totals", s.LogPrefix()), "matches", totalMatch, "blocks", totalBlocks) + } + + return endBlock, nil +} diff --git a/eth/stagedsync/ots_indexer_erc1155_contracts.go b/eth/stagedsync/ots_indexer_erc1155_contracts.go new file mode 100644 index 00000000000..f4d1e7ad8df --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc1155_contracts.go @@ -0,0 +1,55 @@ +package stagedsync + +import ( + "bytes" + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +// This is a Prober that detects if an address contains a contract which implements ERC1155 interface. +// +// It assumes ERC165 detection was already done and it passes the criteria. +type ERC1155Prober struct { + abi *abi.ABI + supportsInterface1155 *[]byte +} + +func NewERC1155Prober() (Prober, error) { + a, err := abi.JSON(bytes.NewReader(otscontracts.ERC165)) + if err != nil { + return nil, err + } + + // Caches predefined supportsInterface() packed calls + siEIP1155, err := a.Pack("supportsInterface", [4]byte{0xd9, 0xb6, 0x7a, 0x26}) + if err != nil { + return nil, err + } + + return &ERC1155Prober{ + abi: &a, + supportsInterface1155: &siEIP1155, + }, nil +} + +func (p *ERC1155Prober) Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, _, _ []byte) (*roaring64.Bitmap, error) { + // supportsInterface(0xd9b67a26) -> ERC1155 interface + res, err := probeContractWithArgs(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.supportsInterface1155, "supportsInterface") + if err != nil { + return nil, err + } + if res == nil || !res[0].(bool) { + return nil, nil + } + + return roaring64.BitmapOf(kv.ADDR_ATTR_ERC1155), nil +} diff --git a/eth/stagedsync/ots_indexer_erc1167_contracts.go b/eth/stagedsync/ots_indexer_erc1167_contracts.go new file mode 100644 index 00000000000..e021ee28459 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc1167_contracts.go @@ -0,0 +1,45 @@ +package stagedsync + +import ( + "bytes" + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" +) + +// This is a Prober that detects if an address contains a contract which implements an ERC1167 minimal proxy +// contract. +// +// It matches the bytecode describe in the specification: https://eips.ethereum.org/EIPS/eip-1167 +type ERC1167Prober struct { +} + +func NewERC1167Prober() (Prober, error) { + return &ERC1167Prober{}, nil +} + +var minimalProxyTemplate = hexutility.Hex2Bytes("363d3d373d3d3d363d73bebebebebebebebebebebebebebebebebebebebe5af43d82803e903d91602b57fd5bf3") + +// TODO: implement support for ERC1167 push optimizations +func (i *ERC1167Prober) Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, _, _ []byte) (*roaring64.Bitmap, error) { + code := ibs.GetCode(addr) + if len(code) != len(minimalProxyTemplate) { + return nil, nil + } + + if !bytes.HasPrefix(code, minimalProxyTemplate[:10]) { + return nil, nil + } + if !bytes.HasSuffix(code, minimalProxyTemplate[30:]) { + return nil, nil + } + + return roaring64.BitmapOf(kv.ADDR_ATTR_ERC1167), nil +} diff --git a/eth/stagedsync/ots_indexer_erc165_contracts.go b/eth/stagedsync/ots_indexer_erc165_contracts.go new file mode 100644 index 00000000000..79c44ed75d0 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc165_contracts.go @@ -0,0 +1,71 @@ +package stagedsync + +import ( + "bytes" + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +// This is a Prober that detects if an address contains a contract which implements ERC165 interface. +// +// It follows the detection mechanism described in the official specification: https://eips.ethereum.org/EIPS/eip-165 +type ERC165Prober struct { + abi *abi.ABI + supportsInterface165 *[]byte + supportsInterfaceFFFFFFFF *[]byte +} + +func NewERC165Prober() (Prober, error) { + // ERC165 + aERC165, err := abi.JSON(bytes.NewReader(otscontracts.ERC165)) + if err != nil { + return nil, err + } + + // Caches predefined supportsInterface() packed calls + siEIP165, err := aERC165.Pack("supportsInterface", [4]byte{0x01, 0xff, 0xc9, 0xa7}) + if err != nil { + return nil, err + } + siFFFFFFFF, err := aERC165.Pack("supportsInterface", [4]byte{0xff, 0xff, 0xff, 0xff}) + if err != nil { + return nil, err + } + + return &ERC165Prober{ + abi: &aERC165, + supportsInterface165: &siEIP165, + supportsInterfaceFFFFFFFF: &siFFFFFFFF, + }, nil +} + +func (p *ERC165Prober) Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, _, _ []byte) (*roaring64.Bitmap, error) { + // supportsInterface(0x01ffc9a7) -> EIP165 interface + res, err := probeContractWithArgs(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.supportsInterface165, "supportsInterface") + if err != nil { + return nil, err + } + if res == nil || !res[0].(bool) { + return nil, nil + } + + // supportsInterface(0xffffffff) -> MUST return false according to EIP165 + res, err = probeContractWithArgs(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.supportsInterfaceFFFFFFFF, "supportsInterface") + if err != nil { + return nil, err + } + if res == nil || res[0].(bool) { + return nil, nil + } + + return roaring64.BitmapOf(kv.ADDR_ATTR_ERC165), nil +} diff --git a/eth/stagedsync/ots_indexer_erc20_721_holders.go b/eth/stagedsync/ots_indexer_erc20_721_holders.go new file mode 100644 index 00000000000..e2d46f93ea1 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc20_721_holders.go @@ -0,0 +1,113 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/log/v3" +) + +// Implements LogIndexerHandler interface in order to index token transfers +// (ERC20/ERC721) +type TransferLogHolderHandler struct { + nft bool + indexBucket string + transfersCollector *etl.Collector +} + +func NewTransferLogHolderHandler(tmpDir string, s *StageState, nft bool, indexBucket string, logger log.Logger) LogIndexerHandler[TransferAnalysisResult] { + transfersCollector := etl.NewCollector(s.LogPrefix(), tmpDir, etl.NewOldestEntryBuffer(etl.BufferOptimalSize), logger) + + return &TransferLogHolderHandler{nft, indexBucket, transfersCollector} +} + +// Add log's ethTx index to from/to addresses indexes +func (h *TransferLogHolderHandler) HandleMatch(match *TxMatchedLogs[TransferAnalysisResult]) { + for _, res := range match.matchResults { + if res.nft != h.nft { + continue + } + + // Register this ethTx into from/to transfer addresses indexes + h.touchIndex(res.from, res.token, match.ethTx) + h.touchIndex(res.to, res.token, match.ethTx) + } +} + +func (h *TransferLogHolderHandler) touchIndex(addr, token common.Address, ethTx uint64) { + // Collect k as holder + token addr in order to get a free dedup from collector; + // it'll be split into v during load phase + k := make([]byte, length.Addr*2) + copy(k, addr.Bytes()) + copy(k[length.Addr:], token.Bytes()) + + // This is the first appearance of address as holder of tokenAddr (in this sync cycle; we don't + // know about the DB, that'll be checked later during load phase); save it (and feed the cache). + // + // Even if cache is evicted, buffer implementation will ensure the key is added only once (saving + // us temp space). + v := make([]byte, length.BlockNum) + binary.BigEndian.PutUint64(v, ethTx) + if err := h.transfersCollector.Collect(k, v); err != nil { + // TODO: change interface to return err + // return startBlock, err + log.Warn("unexpected error", "err", err) + } +} + +func (h *TransferLogHolderHandler) Flush(force bool) error { + return nil +} + +var expectedHolder = hexutil.MustDecode("0xF4445E7218889A91c63d6f95296459Fee6cabC30") +var expectedToken = hexutil.MustDecode("0x44Ce562D8296179630F71680dFb3cA773B4FF5DE") + +func (h *TransferLogHolderHandler) Load(ctx context.Context, tx kv.RwTx) error { + index, err := tx.CursorDupSort(h.indexBucket) + if err != nil { + return err + } + defer index.Close() + + loadFunc := func(k []byte, value []byte, _ etl.CurrentTableReader, next etl.LoadNextFunc) error { + // Buffer will guarantee there is only 1 occurrence of key per provider file; it may appear in + // multiple files or in a previous sync cycle though, so safeguard here + holder := k[:length.Addr] + token := k[length.Addr:] + + v, err := index.SeekBothRange(holder, token) + if err != nil { + return err + } + if v != nil && bytes.HasPrefix(v, token) { + existingEthTx := binary.BigEndian.Uint64(v[length.Addr:]) + newEthTx := binary.BigEndian.Uint64(value) + if newEthTx >= existingEthTx { + return nil + } + } + + newK := k[:length.Addr] + newV := make([]byte, length.Addr+length.BlockNum) + copy(newV, k[length.Addr:]) + copy(newV[length.Addr:], value) + + // smol hack: avoid next(...) bc we rely on dupsort here + return tx.Put(h.indexBucket, newK, newV) + } + if err := h.transfersCollector.Load(tx, h.indexBucket, loadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { + return err + } + + return nil +} + +func (h *TransferLogHolderHandler) Close() { + h.transfersCollector.Close() +} diff --git a/eth/stagedsync/ots_indexer_erc20_721_transfers.go b/eth/stagedsync/ots_indexer_erc20_721_transfers.go new file mode 100644 index 00000000000..7c9e7303636 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc20_721_transfers.go @@ -0,0 +1,81 @@ +package stagedsync + +import ( + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/log/v3" +) + +// Handles ERC20 and ERC721 indexing simultaneously +type MultiLogIndexerHandler[T any] struct { + handlers []LogIndexerHandler[T] +} + +func NewMultiIndexerHandler[T any](handlers ...LogIndexerHandler[T]) *MultiLogIndexerHandler[T] { + return &MultiLogIndexerHandler[T]{ + handlers, + } +} + +func (c *MultiLogIndexerHandler[T]) HandleMatch(output *TxMatchedLogs[T]) { + for _, h := range c.handlers { + h.HandleMatch(output) + } +} + +func (c *MultiLogIndexerHandler[T]) Flush(force bool) error { + for _, h := range c.handlers { + if err := h.Flush(force); err != nil { + return err + } + } + return nil +} + +func (c *MultiLogIndexerHandler[T]) Load(ctx context.Context, tx kv.RwTx) error { + for _, h := range c.handlers { + if err := h.Load(ctx, tx); err != nil { + return err + } + } + return nil +} + +func (c *MultiLogIndexerHandler[T]) Close() { + for _, h := range c.handlers { + h.Close() + } +} + +// Implements LogIndexerHandler interface in order to index token transfers +// (ERC20/ERC721) +type TransferLogIndexerHandler struct { + IndexHandler + nft bool +} + +func NewTransferLogIndexerHandler(tmpDir string, s *StageState, nft bool, indexBucket, counterBucket string, logger log.Logger) LogIndexerHandler[TransferAnalysisResult] { + collector := etl.NewCollector(s.LogPrefix(), tmpDir, etl.NewSortableBuffer(etl.BufferOptimalSize), logger) + bitmaps := map[string]*roaring64.Bitmap{} + + return &TransferLogIndexerHandler{ + &StandardIndexHandler{indexBucket, counterBucket, collector, bitmaps}, + nft, + } +} + +// Add log's ethTx index to from/to addresses indexes +func (h *TransferLogIndexerHandler) HandleMatch(match *TxMatchedLogs[TransferAnalysisResult]) { + for _, res := range match.matchResults { + if res.nft != h.nft { + continue + } + + // Register this ethTx into from/to transfer addresses indexes + h.TouchIndex(res.from, match.ethTx) + h.TouchIndex(res.to, match.ethTx) + } +} diff --git a/eth/stagedsync/ots_indexer_erc20_contracts.go b/eth/stagedsync/ots_indexer_erc20_contracts.go new file mode 100644 index 00000000000..ad923571fd5 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc20_contracts.go @@ -0,0 +1,115 @@ +package stagedsync + +import ( + "bytes" + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +type ERC20Prober struct { + abi *abi.ABI + name *[]byte + symbol *[]byte + decimals *[]byte + junkABI *abi.ABI + junk *[]byte +} + +func NewERC20Prober() (Prober, error) { + // ERC20 + aERC20, err := abi.JSON(bytes.NewReader(otscontracts.ERC20)) + if err != nil { + return nil, err + } + + // Caches name()/symbol()/decimals() packed calls since they don't require + // params + name, err := aERC20.Pack("name") + if err != nil { + return nil, err + } + + symbol, err := aERC20.Pack("symbol") + if err != nil { + return nil, err + } + + decimals, err := aERC20.Pack("decimals") + if err != nil { + return nil, err + } + + // Junk prober + junkABI, err := abi.JSON(bytes.NewReader(otscontracts.Junk)) + if err != nil { + return nil, err + } + junk, err := junkABI.Pack("junkjunkjunk") + if err != nil { + return nil, err + } + + return &ERC20Prober{ + abi: &aERC20, + name: &name, + symbol: &symbol, + decimals: &decimals, + junkABI: &junkABI, + junk: &junk, + }, nil +} + +func (p *ERC20Prober) Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, _, _ []byte) (*roaring64.Bitmap, error) { + // decimals() + res, err, retDecimals := probeContractWithArgs2(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.decimals, "decimals") + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + + // name() + res, err, retName := probeContractWithArgs2(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.name, "name") + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + + // symbol() + res, err, retSymbol := probeContractWithArgs2(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.symbol, "symbol") + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + + // junk + _, err, retJunk := expectRevert(ctx, evm, header, chainConfig, ibs, &addr, p.junk) + if err != nil { + return nil, err + } + + // Detect faulty contracts that return the same junk raw value no matter what you call; + // in this case call a random signature and check if it returns the same as name/symbol/decimals, + // which makes no sense. + if bytes.Equal(retJunk.ReturnData, retName.ReturnData) && + bytes.Equal(retJunk.ReturnData, retSymbol.ReturnData) && + bytes.Equal(retJunk.ReturnData, retDecimals.ReturnData) { + return nil, nil + } + + return roaring64.BitmapOf(kv.ADDR_ATTR_ERC20), nil +} diff --git a/eth/stagedsync/ots_indexer_erc4626_contracts.go b/eth/stagedsync/ots_indexer_erc4626_contracts.go new file mode 100644 index 00000000000..f067812afa5 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc4626_contracts.go @@ -0,0 +1,101 @@ +package stagedsync + +import ( + "bytes" + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +// This is a Prober that detects if an address contains a contract which implements ERC4626 interface. +// +// It assumes ERC20 detection was already done and it passes the criteria. +type ERC4626Prober struct { + abi *abi.ABI + asset *[]byte + totalAssets *[]byte + junkABI *abi.ABI + junk *[]byte +} + +func NewERC4626Prober() (Prober, error) { + // ERC4626 + aIERC4626, err := abi.JSON(bytes.NewReader(otscontracts.IERC4626)) + if err != nil { + return nil, err + } + + // Caches asset()/totalAssets() packed calls since they don't require + // params + asset, err := aIERC4626.Pack("asset") + if err != nil { + return nil, err + } + + totalAssets, err := aIERC4626.Pack("totalAssets") + if err != nil { + return nil, err + } + + // Junk prober + junkABI, err := abi.JSON(bytes.NewReader(otscontracts.Junk)) + if err != nil { + return nil, err + } + junk, err := junkABI.Pack("junkjunkjunk") + if err != nil { + return nil, err + } + + return &ERC4626Prober{ + abi: &aIERC4626, + asset: &asset, + totalAssets: &totalAssets, + junkABI: &junkABI, + junk: &junk, + }, nil +} + +func (p *ERC4626Prober) Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, _, _ []byte) (*roaring64.Bitmap, error) { + // asset() + res, err, retAsset := probeContractWithArgs2(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.asset, "asset") + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + + // totalAssets() + res, err, retTotalAssets := probeContractWithArgs2(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.totalAssets, "totalAssets") + if err != nil { + return nil, err + } + if res == nil { + return nil, nil + } + + // junk + _, err, retJunk := expectRevert(ctx, evm, header, chainConfig, ibs, &addr, p.junk) + if err != nil { + return nil, err + } + + // Detect faulty contracts that return the same junk raw value no matter what you call; + // in this case call a random signature and check if it returns the same as name/symbol/decimals, + // which makes no sense. + if !retJunk.Failed() && bytes.Equal(retJunk.ReturnData, retAsset.ReturnData) && + bytes.Equal(retJunk.ReturnData, retTotalAssets.ReturnData) { + return nil, nil + } + + return roaring64.BitmapOf(kv.ADDR_ATTR_ERC4626), nil +} diff --git a/eth/stagedsync/ots_indexer_erc721_contracts.go b/eth/stagedsync/ots_indexer_erc721_contracts.go new file mode 100644 index 00000000000..6b4f37274a7 --- /dev/null +++ b/eth/stagedsync/ots_indexer_erc721_contracts.go @@ -0,0 +1,74 @@ +package stagedsync + +import ( + "bytes" + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +// This is a Prober that detects if an address contains a contract which implements ERC721 interface. +// +// It assumes ERC165 detection was already done and it passes the criteria. +type ERC721Prober struct { + abi *abi.ABI + supportsInterface721 *[]byte + supportsInterface721MD *[]byte +} + +// TODO: support 721 and 721MD simultaneously +func NewERC721Prober() (Prober, error) { + a, err := abi.JSON(bytes.NewReader(otscontracts.ERC165)) + if err != nil { + return nil, err + } + + // Caches predefined supportsInterface() packed calls + siEIP721, err := a.Pack("supportsInterface", [4]byte{0x80, 0xac, 0x58, 0xcd}) + if err != nil { + return nil, err + } + si721MD, err := a.Pack("supportsInterface", [4]byte{0x5b, 0x5e, 0x13, 0x9f}) + if err != nil { + return nil, err + } + + return &ERC721Prober{ + abi: &a, + supportsInterface721: &siEIP721, + supportsInterface721MD: &si721MD, + }, nil +} + +func (p *ERC721Prober) Probe(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, blockNum uint64, addr common.Address, _, _ []byte) (*roaring64.Bitmap, error) { + bm := roaring64.NewBitmap() + + // supportsInterface(0x80ac58cd) -> ERC721 interface + res, err := probeContractWithArgs(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.supportsInterface721, "supportsInterface") + if err != nil { + return nil, err + } + if res == nil || !res[0].(bool) { + return nil, nil + } + bm.Add(kv.ADDR_ATTR_ERC721) + + // supportsInterface(0x5b5e139f) -> ERC721 Metadata + res, err = probeContractWithArgs(ctx, evm, header, chainConfig, ibs, addr, p.abi, p.supportsInterface721MD, "supportsInterface") + if err != nil { + return nil, err + } + if res != nil && res[0].(bool) { + bm.Add(kv.ADDR_ATTR_ERC721_MD) + } + + return bm, nil +} diff --git a/eth/stagedsync/ots_indexer_generic.go b/eth/stagedsync/ots_indexer_generic.go new file mode 100644 index 00000000000..b2acf4edacc --- /dev/null +++ b/eth/stagedsync/ots_indexer_generic.go @@ -0,0 +1,341 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "time" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/ots/indexer" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" + "golang.org/x/exp/slices" +) + +func NewGenericIndexerUnwinder(targetBucket, counterBucket string, attrs *roaring64.Bitmap) UnwindExecutor { + return func(ctx context.Context, tx kv.RwTx, u *UnwindState, _ services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker) error { + return runUnwind(ctx, tx, isShortInterval, logEvery, u, targetBucket, counterBucket, attrs) + } +} + +func runUnwind(ctx context.Context, tx kv.RwTx, isShortInterval bool, logEvery *time.Ticker, u *UnwindState, targetBucket, counterBucket string, attrs *roaring64.Bitmap) error { + target, err := tx.RwCursorDupSort(targetBucket) + if err != nil { + return err + } + defer target.Close() + + counter, err := tx.RwCursor(counterBucket) + if err != nil { + return err + } + defer counter.Close() + + // The unwind interval is ]u.UnwindPoint, EOF] + startBlock := hexutility.EncodeTs(u.UnwindPoint + 1) + + // Delete all specified address attributes for affected addresses down to + // unwind point + 1 + if attrs != nil { + k, v, err := target.Seek(startBlock) + if err != nil { + return err + } + for k != nil { + addr := common.BytesToAddress(v) + if err := RemoveAttributes(tx, addr, attrs); err != nil { + return err + } + + k, v, err = target.NextDup() + if err != nil { + return err + } + if k == nil { + k, v, err = target.NextNoDup() + if err != nil { + return err + } + } + } + } + + // Delete all block indexes backwards down to unwind point + 1 + unwoundBlock, err := unwindUint64KeyBasedDupTable(target, u.UnwindPoint) + if err != nil { + return err + } + + // Delete all indexer counters backwards down to unwind point + 1 + k, v, err := counter.Last() + if err != nil { + return err + } + for k != nil { + blockNum := binary.BigEndian.Uint64(v) + if blockNum <= u.UnwindPoint { + if blockNum != unwoundBlock { + log.Error(fmt.Sprintf("[%s] Counter index is corrupt; please report as a bug", u.LogPrefix()), "unwindPoint", u.UnwindPoint, "blockNum", blockNum) + return fmt.Errorf("[%s] Counter index is corrupt; please report as a bug: unwindPoint=%v blockNum=%v", u.LogPrefix(), u.UnwindPoint, blockNum) + } + break + } + + if err := counter.DeleteCurrent(); err != nil { + return err + } + + // TODO: replace it by Prev(); investigate why it is not working (dupsort.PrevNoDup() works fine) + k, v, err = counter.Last() + if err != nil { + return err + } + } + + return nil +} + +// Unwind a table whose key is an uint64 (blockNum, ethTx, whatever... we don't care) by +// deleting everything from the end backwards while the key is > specified key. +// +// Usually the unwindTo param is the uunwind point, so its associated record must be preserved, +// hence deleting everything up to unwind point + 1. +func unwindUint64KeyBasedTable(cursor kv.RwCursor, unwindTo uint64) (uint64, error) { + unwoundToKey := uint64(0) + + k, _, err := cursor.Last() + if err != nil { + return 0, err + } + for k != nil { + kAsNum := binary.BigEndian.Uint64(k) + if kAsNum <= unwindTo { + unwoundToKey = kAsNum + break + } + + if err := cursor.DeleteCurrent(); err != nil { + return 0, err + } + + k, _, err = cursor.Prev() + if err != nil { + return 0, err + } + } + + return unwoundToKey, nil +} + +// Unwind a dupsorted table whose key is an uint64 (blockNum, ethTx, whatever... we don't care) by +// deleting everything from the end backwards while the key is > specified key. +// +// Usually the unwindTo param is the uunwind point, so its associated record must be preserved, +// hence deleting everything up to unwind point + 1. +func unwindUint64KeyBasedDupTable(cursor kv.RwCursorDupSort, unwindTo uint64) (uint64, error) { + unwoundToKey := uint64(0) + + k, _, err := cursor.Last() + if err != nil { + return 0, err + } + for k != nil { + kAsNum := binary.BigEndian.Uint64(k) + if kAsNum <= unwindTo { + unwoundToKey = kAsNum + break + } + + if err := cursor.DeleteCurrentDuplicates(); err != nil { + return 0, err + } + + k, _, err = cursor.PrevNoDup() + if err != nil { + return 0, err + } + } + + return unwoundToKey, nil +} + +// Unwind a pair of buckets in the format: +// +// Index table: k: addr+chunkID, v: chunks +// Counter table: k: addr+counter, v: chunkID +func unwindAddress(tx kv.RwTx, target, targetDel kv.RwCursor, counter kv.RwCursorDupSort, indexBucket, counterBucket string, addr common.Address, idx uint64) error { + key := chunkKey(addr.Bytes(), false, idx) + k, v, err := target.Seek(key) + if err != nil { + return err + } + if k == nil || !bytes.HasPrefix(k, addr.Bytes()) { + // that's ok, because for unwind we take the shortcut and cut everything + // onwards at the first occurrence, but there may be further occurrences + // which will just be ignored + return nil + } + + // Skip potential rewrites of same chunk due to multiple matches on later blocks + lastVal := binary.BigEndian.Uint64(v[len(v)-8:]) + if lastVal < idx { + return nil + } + + foundChunk := binary.BigEndian.Uint64(k[length.Addr:]) + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + + for i := 0; i < len(v); i += 8 { + val := binary.BigEndian.Uint64(v[i : i+8]) + if val >= idx { + break + } + bm.Add(val) + } + + // Safe copy + k = slices.Clone(k) + v = slices.Clone(v) + + // Remove all following chunks + for { + if err := targetDel.Delete(k); err != nil { + return err + } + + k, _, err = target.Next() + if err != nil { + return err + } + k = slices.Clone(k) + if k == nil || !bytes.HasPrefix(k, addr.Bytes()) { + break + } + } + + if !bm.IsEmpty() { + // Rewrite the found chunk as the last + newKey := chunkKey(addr.Bytes(), true, 0) + buf := bytes.NewBuffer(nil) + b := make([]byte, 8) + for it := bm.Iterator(); it.HasNext(); { + val := it.Next() + binary.BigEndian.PutUint64(b, val) + buf.Write(b) + } + if err := tx.Put(indexBucket, newKey, buf.Bytes()); err != nil { + return err + } + } else { + // Rewrite the last remaining chunk as the last + k, v, err := target.Prev() + if err != nil { + return err + } + k = slices.Clone(k) + v = slices.Clone(v) + if k != nil && bytes.HasPrefix(k, addr.Bytes()) { + if err := targetDel.Delete(k); err != nil { + return err + } + + binary.BigEndian.PutUint64(k[length.Addr:], ^uint64(0)) + if err := tx.Put(indexBucket, k, v); err != nil { + return err + } + } + } + + // Delete counters backwards up to the chunk found + k, _, err = counter.SeekExact(addr.Bytes()) + if err != nil { + return err + } + k = slices.Clone(k) + if k == nil { + return fmt.Errorf("possible db corruption; can't find bucket=%s addr=%s data", counterBucket, addr) + } + + // Determine if counter is stored in optimized format + c, err := counter.CountDuplicates() + if err != nil { + return err + } + v, err = counter.LastDup() + if err != nil { + return err + } + v = slices.Clone(v) + isSingleChunkOptimized := c == 1 && len(v) == 1 + + // Delete last counter, it'll be replaced on the next step + lastCounter := uint64(0) + if isSingleChunkOptimized { + if err := counter.DeleteCurrent(); err != nil { + return err + } + } else { + for { + if len(v) == 1 { + // DB corrupted + return fmt.Errorf("db possibly corrupted, len(v) == 1: bucket=%s addr=%s k=%s v=%s", counterBucket, addr, hexutility.Encode(k), hexutility.Encode(v)) + } + lastCounter = binary.BigEndian.Uint64(v[:length.Counter]) + chunk := binary.BigEndian.Uint64(v[length.Counter:]) + + if chunk < foundChunk { + break + } + if err := counter.DeleteCurrent(); err != nil { + return err + } + k, v, err = counter.PrevDup() + if err != nil { + return err + } + k = slices.Clone(k) + v = slices.Clone(v) + if k == nil { + lastCounter = 0 + isSingleChunkOptimized = true + break + } + } + } + + // Replace counter + if !bm.IsEmpty() { + newCounter := lastCounter + bm.GetCardinality() + var newValue []byte + if isSingleChunkOptimized && newCounter <= 256 { + newValue = indexer.OptimizedCounterSerializer(newCounter) + } else { + newValue = indexer.LastCounterSerializer(newCounter) + } + if err := tx.Put(counterBucket, addr.Bytes(), newValue); err != nil { + return err + } + } else { + // Rewrite previous counter (if it exists) pointing it to last chunk + if k != nil && !isSingleChunkOptimized { + if err := counter.DeleteCurrent(); err != nil { + return err + } + + binary.BigEndian.PutUint64(v[length.Counter:], ^uint64(0)) + if err := tx.Put(counterBucket, addr.Bytes(), v); err != nil { + return err + } + } + } + + return nil +} diff --git a/eth/stagedsync/ots_indexer_header.go b/eth/stagedsync/ots_indexer_header.go new file mode 100644 index 00000000000..7f34ec926d8 --- /dev/null +++ b/eth/stagedsync/ots_indexer_header.go @@ -0,0 +1,74 @@ +package stagedsync + +import ( + "context" + "fmt" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +type HeaderIndexerHandler interface { + ResourceAwareIndexHandler + HandleMatch(header *types.Header) +} + +// TODO: extract common logic from runIncrementalBlockIndexerExecutor +func runIncrementalHeaderIndexerExecutor(db kv.RoDB, tx kv.RwTx, blockReader services.FullBlockReader, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, ctx context.Context, s *StageState, handler HeaderIndexerHandler) (uint64, error) { + // Tracks how many blocks finished analysis so far + totalBlocks := uint64(0) + + // Tracks how many blocks finished analysis with a match so far + totalMatch := uint64(0) + + // Process control + flushEvery := time.NewTicker(bitmapsFlushEvery) + defer flushEvery.Stop() + + // Iterate over all blocks [startBlock, endBlock] + for blockNum := startBlock; blockNum <= endBlock; blockNum++ { + hash, err := blockReader.CanonicalHash(ctx, tx, blockNum) + if err != nil { + return startBlock, err + } + header, err := blockReader.HeaderByHash(ctx, tx, hash) + if err != nil { + return startBlock, err + } + + totalBlocks++ + totalMatch++ + handler.HandleMatch(header) + + select { + default: + case <-ctx.Done(): + return startBlock, common.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Scanning blocks", s.LogPrefix()), "block", blockNum, "matches", totalMatch, "blocks", totalBlocks) + case <-flushEvery.C: + if err := handler.Flush(false); err != nil { + return startBlock, err + } + } + } + + // Last (forced) flush and batch load (if applicable) + if err := handler.Flush(true); err != nil { + return startBlock, err + } + if err := handler.Load(ctx, tx); err != nil { + return startBlock, err + } + + // Don't print summary if no contracts were analyzed to avoid polluting logs + if !isShortInterval && totalBlocks > 0 { + log.Info(fmt.Sprintf("[%s] Totals", s.LogPrefix()), "matches", totalMatch, "blocks", totalBlocks) + } + + return endBlock, nil +} diff --git a/eth/stagedsync/ots_indexer_log.go b/eth/stagedsync/ots_indexer_log.go new file mode 100644 index 00000000000..a349f12077c --- /dev/null +++ b/eth/stagedsync/ots_indexer_log.go @@ -0,0 +1,344 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "time" + + "github.com/RoaringBitmap/roaring" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" + "github.com/ledgerwatch/erigon/ethdb/cbor" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" + "go.uber.org/atomic" + "golang.org/x/sync/errgroup" +) + +// Given a log entry, answer the question: does the ETH tx it belongs to deserves to be indexed? +// +// The generic parameter T represents the analysis result and is implementation-specific. E.g. it +// can contain which addresses this log entry touches in a token transfer indexer. +// +// An instance of this interface is meant to be reused, so it can contain caches to speed up further +// analysis. +type LogAnalyzer[T any] interface { + // Given a log entry (there may be others in the same tx, here we analyze 1 specific log entry), + // does it match the criteria the implementation is suposed to analyze? + // + // Return nil means it doesn't pass the criteria and it shouldn't be indexed. + Inspect(tx kv.Tx, l *types.Log) (*T, error) +} + +// Handles log indexer lifecycle. +type LogIndexerHandler[T any] interface { + ResourceAwareIndexHandler + + // Given a tx that must be indexed, handles all logs that caused the matching. + HandleMatch(match *TxMatchedLogs[T]) +} + +func runConcurrentLogIndexerExecutor[T any](db kv.RoDB, tx kv.RwTx, blockReader services.FullBlockReader, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, ctx context.Context, s *StageState, analyzer LogAnalyzer[T], handler LogIndexerHandler[T]) (uint64, error) { + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Using concurrent executor", s.LogPrefix())) + } + + // Scans logs bucket + logs, err := tx.Cursor(kv.Log) + if err != nil { + return startBlock, err + } + defer logs.Close() + + // Tracks how many txs finished analysis so far + totalTx := atomic.NewUint64(0) + + // Tracks how many txs finished analysis with a match so far + totalMatch := atomic.NewUint64(0) + + // Each worker processes all logs from 1 ethTx each time + workers := estimate.AlmostAllCPUs() + proberCh := make(chan *TxLogs[T], workers*3) + matchCh := make(chan *TxMatchedLogs[T], workers*3+workers) // must be >= inCh buffer size + n. of workers, otherwise can deadlock + + g, gCtx := errgroup.WithContext(ctx) + for i := 0; i < workers; i++ { + createLogAnalyzerWorker(g, gCtx, db, analyzer, proberCh, matchCh, totalMatch, totalTx) + } + + // Process control + flushEvery := time.NewTicker(bitmapsFlushEvery) + defer flushEvery.Stop() + + // Get all blocks in [startBlock, endBlock] that contain at least 1 TRANSFER_TOPIC occurrence + blocks, err := newBlockBitmapFromTopic(tx, startBlock, endBlock, TRANSFER_TOPIC) + if err != nil { + return startBlock, err + } + defer bitmapdb.ReturnToPool(blocks) + +L: + // Iterate each block that contains a Transfer() event + for it := blocks.Iterator(); it.HasNext(); { + blockNum := uint64(it.Next()) + + // Avoid recalculating txid from the block basetxid for each match + baseTxId, err := blockReader.BaseTxIdForBlock(ctx, tx, blockNum) + if err != nil { + return startBlock, err + } + + // Inspect each tx's logs + logPrefix := hexutility.EncodeTs(blockNum) + for k, v, err := logs.Seek(logPrefix); k != nil && bytes.HasPrefix(k, logPrefix); k, v, err = logs.Next() { + if err != nil { + return startBlock, err + } + + txLogs := newTxLogsFromRaw[T](blockNum, baseTxId, k, v) + LG: + for { + select { + case proberCh <- txLogs: + break LG + case match := <-matchCh: + handler.HandleMatch(match) + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Scanning logs", s.LogPrefix()), "block", blockNum, "matches", totalMatch, "txCount", totalTx, "inCh", len(proberCh), "outCh", len(matchCh)) + } + } + } + + select { + default: + case <-ctx.Done(): + break L + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Scanning logs", s.LogPrefix()), "block", blockNum, "matches", totalMatch, "txCount", totalTx, "inCh", len(proberCh), "outCh", len(matchCh)) + case <-flushEvery.C: + if err := handler.Flush(false); err != nil { + return startBlock, err + } + } + } + + // Close in channel and wait for workers to finish + close(proberCh) + if err := g.Wait(); err != nil { + return startBlock, err + } + + // Close out channel and drain remaining data saving them into db + close(matchCh) + for output := range matchCh { + handler.HandleMatch(output) + } + + // Last (forced) flush and batch load (if applicable) + if err := handler.Flush(true); err != nil { + return startBlock, err + } + if err := handler.Load(ctx, tx); err != nil { + return startBlock, err + } + + // Don't print summary if no contracts were analyzed to avoid polluting logs + if !isShortInterval && totalTx.Load() > 0 { + log.Info(fmt.Sprintf("[%s] Totals", s.LogPrefix()), "matches", totalMatch, "txCount", totalTx) + } + + return endBlock, nil +} + +func runIncrementalLogIndexerExecutor[T any](db kv.RoDB, tx kv.RwTx, blockReader services.FullBlockReader, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, ctx context.Context, s *StageState, analyzer LogAnalyzer[T], handler LogIndexerHandler[T]) (uint64, error) { + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Using incremental executor", s.LogPrefix())) + } + + // Scans logs bucket + logs, err := tx.Cursor(kv.Log) + if err != nil { + return startBlock, err + } + defer logs.Close() + + // Tracks how many txs finished analysis so far + totalTx := atomic.NewUint64(0) + + // Tracks how many txs finished analysis with a match so far + totalMatch := atomic.NewUint64(0) + + // Process control + flushEvery := time.NewTicker(bitmapsFlushEvery) + defer flushEvery.Stop() + + // Get all blocks in [startBlock, endBlock] that contain at least 1 TRANSFER_TOPIC occurrence + blocks, err := newBlockBitmapFromTopic(tx, startBlock, endBlock, TRANSFER_TOPIC) + if err != nil { + return startBlock, err + } + defer bitmapdb.ReturnToPool(blocks) + + // Iterate each block that contains a Transfer() event + for it := blocks.Iterator(); it.HasNext(); { + blockNum := uint64(it.Next()) + + // Avoid recalculating txid from the block basetxid for each match + baseTxId, err := blockReader.BaseTxIdForBlock(ctx, tx, blockNum) + if err != nil { + return startBlock, err + } + + // Inspect each tx's logs + logPrefix := hexutility.EncodeTs(blockNum) + for k, v, err := logs.Seek(logPrefix); k != nil && bytes.HasPrefix(k, logPrefix); k, v, err = logs.Next() { + if err != nil { + return startBlock, err + } + + txLogs := newTxLogsFromRaw[T](blockNum, baseTxId, k, v) + results, err := AnalyzeLogs(tx, analyzer, txLogs.rawLogs) + if err != nil { + return startBlock, err + } + + totalTx.Inc() + if len(results) > 0 { + totalMatch.Inc() + match := &TxMatchedLogs[T]{txLogs, results} + handler.HandleMatch(match) + } + } + + select { + default: + case <-ctx.Done(): + return startBlock, common.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Scanning logs", s.LogPrefix()), "block", blockNum, "matches", totalMatch, "txCount", totalTx) + case <-flushEvery.C: + if err := handler.Flush(false); err != nil { + return startBlock, err + } + } + } + + // Last (forced) flush and batch load (if applicable) + if err := handler.Flush(true); err != nil { + return startBlock, err + } + if err := handler.Load(ctx, tx); err != nil { + return startBlock, err + } + + // Don't print summary if no contracts were analyzed to avoid polluting logs + if !isShortInterval && totalTx.Load() > 0 { + log.Info(fmt.Sprintf("[%s] Totals", s.LogPrefix()), "matches", totalMatch, "txCount", totalTx) + } + + return endBlock, nil +} + +// Gets a bitmap with all blocks [startBlock, endBlock] that contains at least 1 occurrence of +// a topic (== log0). +// +// The returned bitmap MUST be returned to pool after use. +func newBlockBitmapFromTopic(tx kv.Tx, startBlock, endBlock uint64, topic []byte) (*roaring.Bitmap, error) { + allBlocks := bitmapdb.NewBitmap() + allBlocks.AddRange(uint64(startBlock), uint64(endBlock)+1) + + chunkedTransfer, err := bitmapdb.Get(tx, kv.LogTopicIndex, topic, uint32(startBlock), uint32(endBlock)) + if err != nil { + return nil, err + } + allBlocks.And(chunkedTransfer) + + return allBlocks, nil +} + +// Represents a set of all raw logs of 1 transaction that'll be analyzed. +// +// 0 or more logs can contribute for matching and eventual indexing of this +// tx on 0 or more target indexes. +type TxLogs[T any] struct { + blockNum uint64 + ethTx uint64 + // raw logs for 1 tx + rawLogs []byte +} + +// k, v are the raw key/value from kv.Logs bucket. +func newTxLogsFromRaw[T any](blockNum, baseTxId uint64, k, v []byte) *TxLogs[T] { + // idx inside block + txIdx := binary.BigEndian.Uint32(k[length.BlockNum:]) + + // TODO: extract formula function + ethTx := baseTxId + 1 + uint64(txIdx) + + raw := make([]byte, len(v)) + copy(raw, v) + + return &TxLogs[T]{blockNum, ethTx, raw} +} + +// rawLogs contains N encoded logs for 1 tx +func AnalyzeLogs[T any](tx kv.Tx, analyzer LogAnalyzer[T], rawLogs []byte) ([]*T, error) { + var logs types.Logs + if err := cbor.Unmarshal(&logs, bytes.NewReader(rawLogs)); err != nil { + return nil, err + } + + // scan log entries for tx + results := make([]*T, 0) + for _, l := range logs { + res, err := analyzer.Inspect(tx, l) + if err != nil { + return nil, err + } + if res == nil { + continue + } + // TODO: dedup + results = append(results, res) + } + + return results, nil +} + +type TxMatchedLogs[T any] struct { + *TxLogs[T] + matchResults []*T +} + +func createLogAnalyzerWorker[T any](g *errgroup.Group, ctx context.Context, db kv.RoDB, analyzer LogAnalyzer[T], proberCh <-chan *TxLogs[T], matchCh chan<- *TxMatchedLogs[T], totalMatch, txCount *atomic.Uint64) { + g.Go(func() error { + return db.View(ctx, func(tx kv.Tx) error { + for { + // wait for input + txLogs, ok := <-proberCh + if !ok { + break + } + + results, err := AnalyzeLogs(tx, analyzer, txLogs.rawLogs) + if err != nil { + return err + } + + txCount.Inc() + if len(results) > 0 { + totalMatch.Inc() + matchCh <- &TxMatchedLogs[T]{txLogs, results} + } + } + return nil + }) + }) +} diff --git a/eth/stagedsync/ots_stage_generic.go b/eth/stagedsync/ots_stage_generic.go new file mode 100644 index 00000000000..39a7766fe91 --- /dev/null +++ b/eth/stagedsync/ots_stage_generic.go @@ -0,0 +1,242 @@ +package stagedsync + +import ( + "context" + "fmt" + "time" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/wrap" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/eth/stagedsync/stages" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +type ContractAnalyzerCfg struct { + db kv.RwDB + tmpDir string + chainConfig *chain.Config + blockReader services.FullBlockReader + engine consensus.Engine +} + +func StageDbAwareCfg(db kv.RwDB, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine) ContractAnalyzerCfg { + return ContractAnalyzerCfg{ + db, + tmpDir, + chainConfig, + blockReader, + engine, + } +} + +// This is a hint to supress verbose info logs if the stage block range to be executed is <= this number +// of blocks. +const SHORT_RANGE_EXECUTION_THRESHOLD = 16 + +// Defines a stage executor function to be called back by GenericStageForwardFunc. +// +// It should implement the stage business logic. +// +// Implementation should rely on the tx param for DB access. The db param is provided for a specific +// use-case and should be used with caution. +// +// The db param should be used by concurrent implementations that are optimized for the first sync, hence +// there is no risk of trying to read uncommited data. In this case, the implementation can span several +// goroutines to process data saved by a previous stages concurrently. +type StageExecutor = func(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) + +// Defines a stage executor function to be called back by GenericStageUnwindFunc. +// +// It should implement the stage unwind business logic. +// +// Implementation should rely on the tx param for DB access. +type UnwindExecutor = func(ctx context.Context, tx kv.RwTx, u *UnwindState, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker) error + +// This is a template factory function of stage forward execution implementation. +// +// This implementation handles most common tasks performed by stage execution and allows custom logic +// to be plugged via an executor param. +// +// Tasks performed by this generic implementation: +// +// - It handles the lack of parent tx param meaning stage must begin/commit its own db tx. +// - It provides a standard log ticker. +// - It determines the block range so verbose logs may be supressed. +// - It handles execution completion automatically (db save of last successful block). +func GenericStageForwardFunc(ctx context.Context, cfg ContractAnalyzerCfg, parentStage stages.SyncStage, executor StageExecutor) ExecFunc { + return func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { + return genericStageForwardImpl(ctx, cfg, s, txc.Tx, executor, parentStage, logger, false, 0, 0) + } +} + +func GenericStageForwardFuncWithDebug(ctx context.Context, cfg ContractAnalyzerCfg, parentStage stages.SyncStage, executor StageExecutor, _debugStartBlock, _debugEndBlock uint64) ExecFunc { + return func(firstCycle bool, badBlockUnwind bool, s *StageState, u Unwinder, txc wrap.TxContainer, logger log.Logger) error { + if s.BlockNumber > 0 { + return nil + } + return genericStageForwardImpl(ctx, cfg, s, txc.Tx, executor, parentStage, logger, true, _debugStartBlock, _debugEndBlock) + } +} + +func genericStageForwardImpl(ctx context.Context, cfg ContractAnalyzerCfg, s *StageState, tx kv.RwTx, executor StageExecutor, parentStage stages.SyncStage, logger log.Logger, _debug bool, _debugStartBlock, _debugEndBlock uint64) error { + useExternalTx := tx != nil + + if !useExternalTx { + var err error + tx, err = cfg.db.BeginRw(context.Background()) + if err != nil { + return err + } + defer tx.Rollback() + } + + logEvery := time.NewTicker(logInterval) + defer logEvery.Stop() + + // Determine [startBlock, endBlock] + // + // If saved block number == 0, it means stage was never run and it must start at 0, otherwise + // it represents the latest ran block number, so it must start at block+1. + // + // End block is bound to latest run block number from the parent stage. Must not go further. + startBlock := s.BlockNumber + if startBlock > 0 { + startBlock++ + } + endBlock, err := stages.GetStageProgress(tx, parentStage) + if err != nil { + return err + } + + /////////////////////////// + // DEBUG OVERRIDES + if _debug { + startBlock = _debugStartBlock + endBlock = _debugEndBlock + } + /////////////////////////// + + // startBlock > endBlock means parent stage progress was forcefully reset + // just skip this stage silently + if startBlock > endBlock { + return nil + } + + // Don't display verbose start/finish logs on short range executions; given short == <= N blocks + isShortInterval := endBlock-startBlock+1 <= SHORT_RANGE_EXECUTION_THRESHOLD + + /////////////////////////// + // DEBUG OVERRIDES + if _debug { + isShortInterval = false + } + /////////////////////////// + + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Started", s.LogPrefix()), "from", startBlock, "to", endBlock) + } + lastFinishedBlock, err := executor(ctx, cfg.db, tx, !useExternalTx, cfg.tmpDir, cfg.chainConfig, cfg.blockReader, cfg.engine, startBlock, endBlock, isShortInterval, logEvery, s, logger) + if err != nil { + return err + } + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Finished", s.LogPrefix()), "latest", lastFinishedBlock) + } + + if err := s.Update(tx, lastFinishedBlock); err != nil { + return err + } + if !useExternalTx { + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +// This is a template factory function of stage Unwind implementation. +// +// This implementation handles most common tasks performed by unwind and allows custom logic +// to be plugged via an executor param. +// +// Tasks performed by this generic implementation: +// +// - It handles the lack of parent tx param meaning stage must begin/commit its own db tx. +// - It provides a standard log ticker. +// - It determines the block range so verbose logs may be supressed. +// - It handles unwind completion automatically (db saves). +func GenericStageUnwindFunc(ctx context.Context, cfg ContractAnalyzerCfg, executor UnwindExecutor) UnwindFunc { + return func(firstCycle bool, u *UnwindState, s *StageState, txc wrap.TxContainer, logger log.Logger) error { + return GenericStageUnwindImpl(ctx, txc.Tx, cfg, u, executor) + } +} + +// That shouldn't be module exported, but we use this function on integration +// tool for manual unwinds. +func GenericStageUnwindImpl(ctx context.Context, tx kv.RwTx, cfg ContractAnalyzerCfg, u *UnwindState, executor UnwindExecutor) (err error) { + useExternalTx := tx != nil + + if !useExternalTx { + tx, err = cfg.db.BeginRw(ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + logEvery := time.NewTicker(logInterval) + defer logEvery.Stop() + + // Don't display verbose start/finish logs on short range executions; given short == <= N blocks + isShortInterval := u.CurrentBlockNumber-u.UnwindPoint <= SHORT_RANGE_EXECUTION_THRESHOLD + + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Unwind started", u.LogPrefix()), "from", u.CurrentBlockNumber, "to", u.UnwindPoint) + } + if executor == nil { + log.Warn("Unwinder executor is nil; this should only happen on test/dev code, otherwise this msg must be considered a bug") + } else { + if err := executor(ctx, tx, u, cfg.blockReader, isShortInterval, logEvery); err != nil { + return err + } + } + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Unwind finished", u.LogPrefix()), "latest", u.UnwindPoint) + } + + if err = u.Done(tx); err != nil { + return err + } + + if !useExternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil +} + +// This is a no-op implementation of stage pruning function to be used as +// a filler for stages that don't support pruning. +func NoopStagePrune(ctx context.Context, cfg ContractAnalyzerCfg) PruneFunc { + return func(firstCycle bool, p *PruneState, tx kv.RwTx, logger log.Logger) (err error) { + useExternalTx := tx != nil + if !useExternalTx { + tx, err = cfg.db.BeginRw(ctx) + if err != nil { + return err + } + defer tx.Rollback() + } + + if !useExternalTx { + if err = tx.Commit(); err != nil { + return err + } + } + return nil + } +} diff --git a/eth/stagedsync/ots_stages.go b/eth/stagedsync/ots_stages.go new file mode 100644 index 00000000000..f9036afc6a7 --- /dev/null +++ b/eth/stagedsync/ots_stages.go @@ -0,0 +1,166 @@ +package stagedsync + +import ( + "context" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/eth/stagedsync/stages" +) + +// Standard Otterscan V2 stages; if opted-in, they must be inserted before finish stage. +func OtsStages(ctx context.Context, caCfg ContractAnalyzerCfg) []*Stage { + return []*Stage{ + { + ID: stages.OtsContractIndexer, + Description: "Index contract creation", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.Bodies, ContractIndexerExecutor), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsAllContracts, + kv.OtsAllContractsCounter, + nil, + ), + ), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC20Indexer, + Description: "ERC20 token indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsContractIndexer, + NewConcurrentIndexerExecutor( + NewERC20Prober, + kv.OtsAllContracts, + kv.OtsERC20, + kv.OtsERC20Counter, + )), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsERC20, + kv.OtsERC20Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC20), + )), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC165Indexer, + Description: "ERC165 indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsContractIndexer, + NewConcurrentIndexerExecutor( + NewERC165Prober, + kv.OtsAllContracts, + kv.OtsERC165, + kv.OtsERC165Counter, + )), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsERC165, + kv.OtsERC165Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC165), + )), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC721Indexer, + Description: "ERC721 token indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsERC165Indexer, + NewConcurrentIndexerExecutor( + NewERC721Prober, + kv.OtsERC165, + kv.OtsERC721, + kv.OtsERC721Counter, + )), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsERC721, + kv.OtsERC721Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC721, kv.ADDR_ATTR_ERC721_MD), + )), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC1155Indexer, + Description: "ERC1155 token indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsERC165Indexer, + NewConcurrentIndexerExecutor( + NewERC1155Prober, + kv.OtsERC165, + kv.OtsERC1155, + kv.OtsERC1155Counter, + )), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsERC1155, + kv.OtsERC1155Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC1155), + )), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC1167Indexer, + Description: "ERC1167 proxy indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsContractIndexer, + NewConcurrentIndexerExecutor( + NewERC1167Prober, + kv.OtsAllContracts, + kv.OtsERC1167, + kv.OtsERC1167Counter, + )), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsERC1167, + kv.OtsERC1167Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC1167), + )), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC4626Indexer, + Description: "ERC4626 token indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsERC20Indexer, + NewConcurrentIndexerExecutor( + NewERC4626Prober, + kv.OtsERC20, + kv.OtsERC4626, + kv.OtsERC4626Counter, + )), + Unwind: GenericStageUnwindFunc(ctx, caCfg, + NewGenericIndexerUnwinder( + kv.OtsERC4626, + kv.OtsERC4626Counter, + roaring64.BitmapOf(kv.ADDR_ATTR_ERC4626), + )), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC20And721Transfers, + Description: "ERC20/721 token transfer indexer", + // Binds itself to ERC721 contract classifier as the parent stage on purpose to ensure + // both ERC20 and ERC721 stages are executed. + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsERC721Indexer, ERC20And721TransferIndexerExecutor), + Unwind: GenericStageUnwindFunc(ctx, caCfg, NewGenericLogIndexerUnwinder()), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsERC20And721Holdings, + Description: "ERC20/721 token holdings indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.OtsERC721Indexer, ERC20And721HolderIndexerExecutor), + Unwind: GenericStageUnwindFunc(ctx, caCfg, NewGenericLogHoldingsUnwinder()), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsBlocksRewarded, + Description: "Blocks rewarded indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.Bodies, BlocksRewardedExecutor), + Unwind: GenericStageUnwindFunc(ctx, caCfg, NewGenericBlockIndexerUnwinder(kv.OtsBlocksRewardedIndex, kv.OtsBlocksRewardedCounter, RunBlocksRewardedBlockUnwind)), + Prune: NoopStagePrune(ctx, caCfg), + }, + { + ID: stages.OtsWithdrawals, + Description: "CL withdrawals indexer", + Forward: GenericStageForwardFunc(ctx, caCfg, stages.Bodies, WithdrawalsExecutor), + Unwind: GenericStageUnwindFunc(ctx, caCfg, NewGenericBlockIndexerUnwinder(kv.OtsWithdrawalsIndex, kv.OtsWithdrawalsCounter, RunWithdrawalsBlockUnwind)), + Prune: NoopStagePrune(ctx, caCfg), + }, + } +} diff --git a/eth/stagedsync/ots_stg_executor_all_contracts_indexer.go b/eth/stagedsync/ots_stg_executor_all_contracts_indexer.go new file mode 100644 index 00000000000..7559dcb0aa8 --- /dev/null +++ b/eth/stagedsync/ots_stg_executor_all_contracts_indexer.go @@ -0,0 +1,383 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "time" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/common" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/systemcontracts" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +// This executor indexes contract creation. +// +// It produces as result the bucket OtsDeployments with: +// +// - key: blockNum (uint64) +// - value: address ([20]byte) + incarnation (uint64) +// +// This bucket serves as a starting point to all contract classifiers. +// +// It follows 2 different strategies, the first one "firstSyncStrategy" is optimized +// for traversing the entire DB and create the entire bucket from existing data. But it works +// only the first time. +// +// The next runs use the "continuousStrategy", which for each block traverses the account changeset +// to detect new contract deployments. +func ContractIndexerExecutor(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) { + if startBlock == 0 && isInternalTx { + return firstSyncContractExecutor(tx, tmpDir, chainConfig, blockReader, engine, startBlock, endBlock, isShortInterval, logEvery, ctx, s, logger) + } + return incrementalContractIndexer(tx, tmpDir, chainConfig, blockReader, engine, startBlock, endBlock, isShortInterval, logEvery, ctx, s, logger) +} + +// This strategy must be run ONLY the first time the contract indexer is run. That's because +// this stage traverses PlainContractCode bucket and indexes all existing addresses that contain +// a deployed contract. +// +// That works only the first time, because it is not unwind friendly, however it's more efficient +// than the general strategy of traversing account change sets. +func firstSyncContractExecutor(tx kv.RwTx, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, ctx context.Context, s *StageState, logger log.Logger) (uint64, error) { + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Using concurrent executor", s.LogPrefix())) + } + + plainCC, err := tx.Cursor(kv.PlainContractCode) + if err != nil { + return startBlock, err + } + defer plainCC.Close() + + accHistory, err := tx.Cursor(kv.E2AccountsHistory) + if err != nil { + return startBlock, err + } + defer accHistory.Close() + + contractCount := 0 + reader := state.NewPlainState(tx, startBlock, systemcontracts.SystemContractCodeLookup[chainConfig.ChainName]) + + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + + contractCollector := etl.NewCollector(s.LogPrefix(), tmpDir, etl.NewSortableBuffer(etl.BufferOptimalSize), logger) + defer contractCollector.Close() + + // Iterate through all deployed contracts; identify the block when it was deployed and + // trace it. + // + // We are only interested in the key set (address+incarnation) here. + stopped := false + for k, _, err := plainCC.First(); k != nil && !stopped; k, _, err = plainCC.Next() { + if err != nil { + return startBlock, err + } + + addr := libcommon.BytesToAddress(k[:length.Addr]) + incarnation := binary.BigEndian.Uint64(k[length.Addr:]) + + blockFound, err := locateBlock(accHistory, addr, incarnation, stopped, bm, reader) + if err != nil { + return startBlock, err + } + // TODO: review; use blockFound == 0 to signal skip + if blockFound == 0 { + continue + } + + contractCount++ + + // Insert contract deployment entry + newK, newV := contractMatchKeyPair(blockFound, addr, incarnation) + if err := contractCollector.Collect(newK, newV); err != nil { + return startBlock, err + } + + select { + default: + case <-ctx.Done(): + stopped = true + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s]", s.LogPrefix()), "addr", addr, "incarnation", incarnation, "contractCount", contractCount) + } + } + + if stopped { + return startBlock, libcommon.ErrStopped + } + + if err := contractCollector.Load(tx, kv.OtsAllContracts, etl.IdentityLoadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { + return startBlock, err + } + log.Info(fmt.Sprintf("[%s] Contract count", s.LogPrefix()), "count", contractCount) + + if err := fillCounterBucket(tx, startBlock, endBlock, kv.OtsAllContracts, kv.OtsAllContractsCounter, logEvery, ctx, s); err != nil { + return startBlock, err + } + return endBlock, nil +} + +func locateBlock(accHistory kv.Cursor, addr libcommon.Address, incarnation uint64, stopped bool, bm *roaring64.Bitmap, reader *state.PlainState) (uint64, error) { + kh, vh, err := accHistory.Seek(addr.Bytes()) + if err != nil { + return 0, err + } + if !bytes.HasPrefix(kh, addr.Bytes()) { + return 0, nil + // TODO: fix contracts appearing on PlainContractCode, but not in history + // e.g.: k=0x026950001a85dd75b6a5f17b94c35a2feaeb2421ffffffffffffffff addr=0x02694FebD8d726465976BEE5cE0a9Ef5f72b577f + // return startBlock, nil + } + + // Given address/incarnation, locate in which block it was deployed + blockFound := uint64(0) + for !stopped && blockFound == 0 { + if _, err := bm.ReadFrom(bytes.NewReader(vh)); err != nil { + return 0, err + } + + // default case + reader.SetBlockNr(bm.Maximum() + 1) + acc, err := reader.ReadAccountData(addr) + if err != nil { + return 0, err + } + + if acc == nil || acc.Incarnation == 0 { + // maybe the contract/incarnation isn't deployed yet, but it may have been + // selfdestroyed, can't say for sure, so linear search the entire shard + blockFound, err = findBlockInsideShard(reader, bm, addr, incarnation) + if err != nil { + return 0, err + } + if blockFound != 0 { + break + } + } else if acc.Incarnation >= incarnation { + // found shard + break + } + + kh, vh, err = accHistory.Next() + if err != nil { + return 0, err + } + if !bytes.HasPrefix(kh, addr.Bytes()) { + return 0, nil + } + } + + // Locate block inside shard + if blockFound == 0 { + blockFound, err = findBlockInsideShard(reader, bm, addr, incarnation) + if err != nil { + return 0, err + } + } + + return blockFound, nil +} + +// Encode data into the expected standard format for contract match +// tables. +// +// blockNum uint64 -> address [20]byte + [incarnation uint64 (ONLY if != 1)] +func contractMatchKeyPair(blockNum uint64, addr libcommon.Address, incarnation uint64) ([]byte, []byte) { + k := hexutility.EncodeTs(blockNum) + + var v []byte + if incarnation != 1 { + v = make([]byte, length.Addr+common.IncarnationLength) + copy(v, addr.Bytes()) + binary.BigEndian.PutUint64(v[length.Addr:], incarnation) + } else { + v = make([]byte, length.Addr) + copy(v, addr.Bytes()) + } + + return k, v +} + +// Given a shard of accounts history, locate which block the incarnation was deployed +// by linear searching all touched blocks and comparing the state before/after each one. +// +// TODO: return DB inconsistency error if can't find block inside shard +func findBlockInsideShard(reader *state.PlainState, bm *roaring64.Bitmap, addr libcommon.Address, incarnation uint64) (uint64, error) { + it := bm.Iterator() + for it.HasNext() { + blockNum := it.Next() + reader.SetBlockNr(blockNum + 1) + acc, err := reader.ReadAccountData(addr) + if err != nil { + return 0, err + } + if acc != nil && acc.Incarnation >= incarnation { + return blockNum, nil + } + } + + return 0, nil +} + +func incrementalContractIndexer(tx kv.RwTx, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, ctx context.Context, s *StageState, logger log.Logger) (uint64, error) { + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Using incremental executor", s.LogPrefix())) + } + + acs, err := tx.CursorDupSort(kv.AccountChangeSet) + if err != nil { + return startBlock, err + } + defer acs.Close() + + newContractCount := 0 + reader := state.NewPlainState(tx, startBlock, systemcontracts.SystemContractCodeLookup[chainConfig.ChainName]) + currentBlockNumber := startBlock + for ; currentBlockNumber <= endBlock; currentBlockNumber++ { + key := hexutility.EncodeTs(currentBlockNumber) + k, v, err := acs.Seek(key) + if err != nil { + return startBlock, err + } + + // Iterate through block changeset, identify those entries whose codehash is empty; + // empty codehash == previous state is EOA, next state might be a deployed contract + for ; bytes.Equal(k, key); k, v, err = acs.NextDup() { + if err != nil { + return startBlock, err + } + + addr := libcommon.BytesToAddress(v[:length.Addr]) + + reader.SetBlockNr(currentBlockNumber) + prevBlockAcc, err := reader.ReadAccountData(addr) + if err != nil { + return startBlock, err + } + // acc == nil: state didn't exist yet; empty codehash: no contract yet + if prevBlockAcc != nil && !prevBlockAcc.IsEmptyCodeHash() { + // contract already deployed at this address; ignore + continue + } + + reader.SetBlockNr(currentBlockNumber + 1) + currBlockAcc, err := reader.ReadAccountData(addr) + if err != nil { + return startBlock, err + } + // acc == nil: was selfdestructed here; !empty codehash: was deployed here + if currBlockAcc != nil && !currBlockAcc.IsEmptyCodeHash() { + // detected contract deployment at this address + newK, newV := contractMatchKeyPair(currentBlockNumber, addr, currBlockAcc.Incarnation) + if err := tx.Put(kv.OtsAllContracts, newK, newV); err != nil { + return startBlock, err + } + + newContractCount++ + } + } + + select { + default: + case <-ctx.Done(): + return startBlock, libcommon.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s]", s.LogPrefix()), "currentBlock", currentBlockNumber, "contractCount", newContractCount) + } + } + + if newContractCount > 0 { + log.Info(fmt.Sprintf("[%s] Found new contracts", s.LogPrefix()), "count", newContractCount) + } + + if err := fillCounterBucket(tx, startBlock, endBlock, kv.OtsAllContracts, kv.OtsAllContractsCounter, logEvery, ctx, s); err != nil { + return startBlock, err + } + return endBlock, nil +} + +func fillCounterBucket(tx kv.RwTx, startBlock, endBlock uint64, bucket string, counterBucket string, logEvery *time.Ticker, ctx context.Context, s *StageState) error { + source, err := tx.CursorDupSort(bucket) + if err != nil { + return err + } + defer source.Close() + + counter, err := tx.Cursor(counterBucket) + if err != nil { + return err + } + defer counter.Close() + + // Determine current total + lastCounter, _, err := counter.Last() + if err != nil { + return err + } + currentCount := uint64(0) + if lastCounter != nil { + currentCount = binary.BigEndian.Uint64(lastCounter) + } + + // Traverse [startBlock, endBlock] + blockKey := hexutility.EncodeTs(startBlock) + k, _, err := source.Seek(blockKey) + if err != nil { + return err + } + if k == nil { + return nil + } + currentBlock := binary.BigEndian.Uint64(k) + + for currentBlock <= endBlock { + // Accumulate found contracts + n, err := source.CountDuplicates() + if err != nil { + return err + } + + // Add count of contracts AFTER the block + // k: cummulative count before block + // v: block + currentCount += n + counterKey := hexutility.EncodeTs(currentCount) + if err := tx.Put(counterBucket, counterKey, k); err != nil { + return err + } + + select { + default: + case <-ctx.Done(): + return libcommon.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Updating counters", s.LogPrefix()), "currentBlock", currentBlock, "count", currentCount) + } + + // Next block + k, _, err = source.NextNoDup() + if err != nil { + return err + } + if k == nil { + break + } + currentBlock = binary.BigEndian.Uint64(k) + } + + return nil +} diff --git a/eth/stagedsync/ots_stg_executor_blocks_rewarded_indexer.go b/eth/stagedsync/ots_stg_executor_blocks_rewarded_indexer.go new file mode 100644 index 00000000000..61a0a50ffd1 --- /dev/null +++ b/eth/stagedsync/ots_stg_executor_blocks_rewarded_indexer.go @@ -0,0 +1,41 @@ +package stagedsync + +import ( + "context" + "time" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func BlocksRewardedExecutor(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) { + blocksRewardedHandler := NewBlocksRewardedIndexerHandler(tmpDir, s, logger) + defer blocksRewardedHandler.Close() + + return runIncrementalHeaderIndexerExecutor(db, tx, blockReader, startBlock, endBlock, isShortInterval, logEvery, ctx, s, blocksRewardedHandler) +} + +// Implements HeaderIndexerHandler interface in order to index blocks rewarded +type BlocksRewardedIndexerHandler struct { + IndexHandler +} + +func NewBlocksRewardedIndexerHandler(tmpDir string, s *StageState, logger log.Logger) HeaderIndexerHandler { + collector := etl.NewCollector(s.LogPrefix(), tmpDir, etl.NewSortableBuffer(etl.BufferOptimalSize), logger) + bitmaps := map[string]*roaring64.Bitmap{} + + return &BlocksRewardedIndexerHandler{ + &StandardIndexHandler{kv.OtsBlocksRewardedIndex, kv.OtsBlocksRewardedCounter, collector, bitmaps}, + } +} + +// Index fee recipient address -> blockNum +func (h *BlocksRewardedIndexerHandler) HandleMatch(header *types.Header) { + h.TouchIndex(header.Coinbase, header.Number.Uint64()) +} diff --git a/eth/stagedsync/ots_stg_executor_concurrent_indexer.go b/eth/stagedsync/ots_stg_executor_concurrent_indexer.go new file mode 100644 index 00000000000..9ac63800a31 --- /dev/null +++ b/eth/stagedsync/ots_stg_executor_concurrent_indexer.go @@ -0,0 +1,587 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "sync" + "time" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/core" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/systemcontracts" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/core/vm/evmtypes" + "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +// This is a dual strategy StageExecutor which indexes deployed contracts based on a criteria determined +// by a Prober. +// +// It also works as a filter, taking contracts from sourceBucket -> Prober -> write on targetBucket +// + counterBucket. +// +// During the first sync, it runs the indexer concurrently. After that, during the following syncs, +// it runs it single-threadly, incrementaly. +func NewConcurrentIndexerExecutor(proberFactory ProberFactory, sourceBucket, targetBucket, counterBucket string) StageExecutor { + return func(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) { + if startBlock == 0 && isInternalTx { + return runExecutorConcurrently(ctx, db, tx, chainConfig, blockReader, engine, startBlock, endBlock, isShortInterval, logEvery, s, proberFactory, sourceBucket, targetBucket, counterBucket) + } + return runExecutorIncrementally(ctx, tx, chainConfig, blockReader, engine, startBlock, endBlock, isShortInterval, logEvery, s, proberFactory, sourceBucket, targetBucket, counterBucket) + } +} + +func runExecutorConcurrently(ctx context.Context, db kv.RoDB, tx kv.RwTx, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, proberFactory ProberFactory, sourceBucket, targetBucket, counterBucket string) (uint64, error) { + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Using concurrent executor", s.LogPrefix())) + } + + source, err := tx.CursorDupSort(sourceBucket) + if err != nil { + return startBlock, err + } + defer source.Close() + + target, err := tx.CursorDupSort(targetBucket) + if err != nil { + return startBlock, err + } + defer target.Close() + + counter, err := tx.Cursor(counterBucket) + if err != nil { + return startBlock, err + } + defer counter.Close() + + // Restore last counter + kct, vct, err := counter.Last() + if err != nil { + return startBlock, err + } + lastTotal := uint64(0) + if kct != nil { + lastTotal = binary.BigEndian.Uint64(kct) + } + + // Indexer/counter sanity check + kidx, _, err := target.Last() + if err != nil { + return startBlock, err + } + if !bytes.Equal(kidx, vct) { + return startBlock, fmt.Errorf("bucket doesn't match counterBucket: bucket=%v counter=%v blockNum=%v counterBlockNum=%v", targetBucket, counterBucket, binary.BigEndian.Uint64(kidx), binary.BigEndian.Uint64(vct)) + } + + // Loop over [startBlock, endBlock] + k, v, err := source.Seek(hexutility.EncodeTs(startBlock)) + if err != nil { + return startBlock, err + } + + // No data after startBlock in the source bucket; given endBlock comes from source stage's, + // assume no data up to endBlock. + if k == nil { + return endBlock, nil + } + + // First available data in the source bucket contains a block after the parent stage's endBlock, + // which comes from parent stage's current block; this may indicate a bug in the code (db inconsistency) + // or parent block number was set without proper db cleaning (wrong unwind). + blockNum := binary.BigEndian.Uint64(k[:length.BlockNum]) + if blockNum > endBlock { + return startBlock, fmt.Errorf("found data for block %d in sourceBucket %s, but stage endBlock is %d", blockNum, sourceBucket, endBlock) + } + + // Setup concurrent probers support + workers := estimate.AlmostAllCPUs() + + // Insert sourceBucket data into this channel and workers will probe them + proberCh := make(chan *sourceData, workers*3) + + // Workers will post matched data here + // + // must be >= proberCh buffer size + n. of workers, otherwise can deadlock + matchesCh := make(chan *matchedData, workers*3+workers) + + // Initialize N workers whose job is to read from proberCh, probe the address, + // and in case of match, post results on matchesCh. + var wg sync.WaitGroup + wg.Add(workers) + for i := 0; i < workers; i++ { + prober, err := proberFactory() + if err != nil { + return startBlock, err + } + createExecutor(ctx, db, blockReader, chainConfig, engine, prober, proberCh, matchesCh, &wg) + } + + totalProbed := 0 + totalMatches := 0 + data := NewSourceData(blockNum, k, v) + +L: + // Outer loop handles sourceBucket DB reading -> proberCh (to be executed concurrently) + // and matchesCh -> targetBucket DB writes. + for { + select { + case proberCh <- data: + totalProbed++ + + // Compute next input for insertion into input channel + k, v, err = source.NextDup() + if err != nil { + return startBlock, err + } + if k == nil { + // Next block + k, v, err = source.NextNoDup() + if err != nil { + return startBlock, err + } + + if k == nil { + break L + } + } + + blockNum = binary.BigEndian.Uint64(k[:length.BlockNum]) + if blockNum > endBlock { + break L + } + + data = NewSourceData(blockNum, k, v) + case match := <-matchesCh: + // Extract next matched data and save into db + if err := tx.Put(targetBucket, match.k, match.v); err != nil { + return startBlock, err + } + + // Save attributes + if err := AddOrUpdateAttributes(tx, match.addr, match.attrs); err != nil { + return startBlock, err + } + + totalMatches++ + case <-ctx.Done(): + return startBlock, common.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Probing", s.LogPrefix()), "block", blockNum, "totalMatches", totalMatches, "totalProbed", totalProbed, "inCh", len(proberCh), "outCh", len(matchesCh)) + } + } + + // Close prober channel and wait for workers to finish + close(proberCh) + wg.Wait() + + // Close out channel and drain remaining data saving them into db; we are + // guaranteed no more probing data will be posted here. + close(matchesCh) + for match := range matchesCh { + if err := tx.Put(targetBucket, match.k, match.v); err != nil { + return startBlock, err + } + totalMatches++ + } + + // Rewind target cursor and write counters [startBlock, endBlock] + k, _, err = target.Seek(hexutility.EncodeTs(startBlock)) + if err != nil { + return startBlock, err + } + + currBlockTotal := lastTotal + for k != nil { + blockNum := binary.BigEndian.Uint64(k[:length.BlockNum]) + count, err := target.CountDuplicates() + if err != nil { + return startBlock, err + } + currBlockTotal += count + + // Persist accumulated counter for current block + if err := writeCounter(tx, counterBucket, currBlockTotal, blockNum); err != nil { + return startBlock, err + } + + // Next block containing matches + k, _, err = target.NextNoDup() + if err != nil { + return startBlock, err + } + } + + // Don't print summary if no contracts were analyzed to avoid polluting logs + if !isShortInterval && totalProbed > 0 { + log.Info(fmt.Sprintf("[%s] Totals", s.LogPrefix()), "totalMatches", totalMatches, "totalProbed", totalProbed) + } + + return endBlock, nil +} + +func runExecutorIncrementally(ctx context.Context, tx kv.RwTx, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, proberFactory ProberFactory, sourceBucket, targetBucket, counterBucket string) (uint64, error) { + if !isShortInterval { + log.Info(fmt.Sprintf("[%s] Using incremental executor", s.LogPrefix())) + } + + // Open required cursors + source, err := tx.CursorDupSort(sourceBucket) + if err != nil { + return startBlock, err + } + defer source.Close() + + target, err := tx.CursorDupSort(targetBucket) + if err != nil { + return startBlock, err + } + defer target.Close() + + counter, err := tx.Cursor(counterBucket) + if err != nil { + return startBlock, err + } + defer counter.Close() + + // Restore last counter + kct, vct, err := counter.Last() + if err != nil { + return startBlock, err + } + lastTotal := uint64(0) + if kct != nil { + lastTotal = binary.BigEndian.Uint64(kct) + } + + // Indexer/counter sanity check + kidx, _, err := target.Last() + if err != nil { + return startBlock, err + } + if !bytes.Equal(kidx, vct) { + return startBlock, fmt.Errorf("bucket doesn't match counterBucket: bucket=%v counter=%v blockNum=%v counterBlockNum=%v", targetBucket, counterBucket, binary.BigEndian.Uint64(kidx), binary.BigEndian.Uint64(vct)) + } + + prober, err := proberFactory() + if err != nil { + return startBlock, err + } + + getHeader := func(hash common.Hash, n uint64) *types.Header { + h, err := blockReader.Header(ctx, tx, hash, n) + if err != nil { + log.Error("Can't get block hash by number", "number", n, "only-canonical", false) + return nil + } + return h + } + + ex := executor{ + ctx: ctx, + tx: tx, + blockReader: blockReader, + chainConfig: chainConfig, + getHeader: getHeader, + engine: engine, + prober: prober, + } + + // Loop over [startBlock, endBlock] + k, v, err := source.Seek(hexutility.EncodeTs(startBlock)) + if err != nil { + return startBlock, err + } + + // No new data in the [startBlock, ...] interval in the source bucket; since endBlock comes + // from sourceBucket's stage, assumes the entire interval is done. + if k == nil { + return endBlock, nil + } + + // First available data in the source bucket contains a block after the parent stage's endBlock, + // which comes from parent stage's current block; this may indicate a bug in the code (db inconsistency) + // or parent block number was set without proper db cleaning (wrong unwind). + blockNum := binary.BigEndian.Uint64(k[:length.BlockNum]) + if blockNum > endBlock { + return startBlock, fmt.Errorf("found data for block %d in sourceBucket %s, but stage endBlock is %d", blockNum, sourceBucket, endBlock) + } + + prevBlockTotal := lastTotal + currBlockTotal := lastTotal + + totalProbed := 0 + for { + totalProbed++ + addr := common.BytesToAddress(v) + attrs, err := ex.ResetAndProbe(blockNum, addr, k, v) + if err != nil { + // Ignore execution errors on purpose + log.Warn("ignored error", "err", err) + } + + // Save match + if attrs != nil { + if err := tx.Put(targetBucket, k, v); err != nil { + return startBlock, err + } + + // Save attributes + if err := AddOrUpdateAttributes(tx, addr, attrs); err != nil { + return startBlock, err + } + + currBlockTotal++ + } + + // Compute next input for insertion into input channel + k, v, err = source.NextDup() + if err != nil { + return startBlock, err + } + if k == nil { + // Next block + k, v, err = source.NextNoDup() + if err != nil { + return startBlock, err + } + + // EOF + if k == nil { + if currBlockTotal > prevBlockTotal { + // Cut block counter and save accumulated totals + if err := writeCounter(tx, counterBucket, currBlockTotal, blockNum); err != nil { + return startBlock, err + } + } + break + } + } + + newBlockNum := binary.BigEndian.Uint64(k[:length.BlockNum]) + if newBlockNum != blockNum { + if currBlockTotal > prevBlockTotal { + // Cut block counter and save accumulated totals + if err := writeCounter(tx, counterBucket, currBlockTotal, blockNum); err != nil { + return startBlock, err + } + prevBlockTotal = currBlockTotal + } + + if newBlockNum > endBlock { + break + } + blockNum = newBlockNum + } + + select { + default: + case <-ctx.Done(): + return startBlock, common.ErrStopped + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Probing", s.LogPrefix()), "block", blockNum, "totalMatches", currBlockTotal-lastTotal, "totalProbed", totalProbed) + } + } + + // Don't print summary if no contracts were analyzed to avoid polluting logs + if !isShortInterval && totalProbed > 0 { + log.Info(fmt.Sprintf("[%s] Totals", s.LogPrefix()), "totalMatches", currBlockTotal-lastTotal, "totalProbed", totalProbed) + } + + return endBlock, nil +} + +type sourceData struct { + blockNum uint64 + addr common.Address + k []byte + v []byte +} + +func NewSourceData(blockNum uint64, k, v []byte) *sourceData { + addr := common.BytesToAddress(v) + ck := make([]byte, len(k)) + cv := make([]byte, len(v)) + copy(ck, k) + copy(cv, v) + return &sourceData{blockNum, addr, ck, cv} +} + +type matchedData struct { + blockNum uint64 + addr common.Address + k []byte + v []byte + attrs *roaring64.Bitmap +} + +func AddOrUpdateAttributes(tx kv.RwTx, addr common.Address, attrs *roaring64.Bitmap) error { + if attrs == nil { + log.Warn("attrs bitmap shouldn't be nil") + } + + addrKey := addr.Bytes() + a, err := tx.GetOne(kv.OtsAddrAttributes, addrKey) + if err != nil { + return err + } + + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + + if a != nil { + if _, err := bm.ReadFrom(bytes.NewReader(a)); err != nil { + return err + } + } + bm.Or(attrs) + + bm.RunOptimize() + b, err := bm.ToBytes() + if err != nil { + return err + } + if err := tx.Put(kv.OtsAddrAttributes, addrKey, b); err != nil { + return err + } + + return nil +} + +func RemoveAttributes(tx kv.RwTx, addr common.Address, attrs *roaring64.Bitmap) error { + if attrs == nil { + log.Warn("attrs bitmap shouldn't be nil") + } + + addrKey := addr.Bytes() + a, err := tx.GetOne(kv.OtsAddrAttributes, addrKey) + if err != nil { + return err + } + if a == nil { + return nil + } + + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + + if _, err := bm.ReadFrom(bytes.NewReader(a)); err != nil { + return err + } + bm.AndNot(attrs) + + // If this attribute removal leads to no attributes set, remove the record entirely + if bm.IsEmpty() { + return tx.Delete(kv.OtsAddrAttributes, addrKey) + } + + // Optimize and update + bm.RunOptimize() + b, err := bm.ToBytes() + if err != nil { + return err + } + if err := tx.Put(kv.OtsAddrAttributes, addrKey, b); err != nil { + return err + } + + return nil +} + +func writeCounter(tx kv.RwTx, counterBucket string, counter, blockNum uint64) error { + k := hexutility.EncodeTs(counter) + v := hexutility.EncodeTs(blockNum) + return tx.Put(counterBucket, k, v) +} + +type executor struct { + ctx context.Context + tx kv.Tx + blockReader services.FullBlockReader + chainConfig *chain.Config + getHeader func(common.Hash, uint64) *types.Header + engine consensus.Engine + prober Prober +} + +func createExecutor(ctx context.Context, db kv.RoDB, blockReader services.FullBlockReader, chainConfig *chain.Config, engine consensus.Engine, prober Prober, proberCh <-chan *sourceData, matchesCh chan<- *matchedData, wg *sync.WaitGroup) { + go func() { + defer wg.Done() + + tx, err := db.BeginRo(ctx) + if err != nil { + // TODO: handle error + return + } + defer tx.Rollback() + + getHeader := func(hash common.Hash, n uint64) *types.Header { + h, err := blockReader.Header(ctx, tx, hash, n) + if err != nil { + log.Error("Can't get block hash by number", "number", n, "only-canonical", false) + return nil + } + return h + } + + ex := executor{ + ctx: ctx, + tx: tx, + blockReader: blockReader, + chainConfig: chainConfig, + getHeader: getHeader, + engine: engine, + prober: prober, + } + + for { + // wait for input + source, ok := <-proberCh + if !ok { + break + } + + attrs, err := ex.ResetAndProbe(source.blockNum, source.addr, source.k, source.v) + if err != nil { + // Ignore execution errors on purpose + log.Warn("ignored error", "err", err) + } + if attrs != nil { + matchesCh <- &matchedData{source.blockNum, source.addr, source.k, source.v, attrs} + } + } + }() +} + +func (ex *executor) ResetAndProbe(blockNumber uint64, addr common.Address, k, v []byte) (*roaring64.Bitmap, error) { + header, err := ex.blockReader.HeaderByNumber(ex.ctx, ex.tx, blockNumber) + if err != nil { + return nil, err + } + if header == nil { + // TODO: corrupted? + log.Warn("couldn't find header", "blockNum", blockNumber) + return nil, nil + } + + stateReader := state.NewPlainState(ex.tx, blockNumber+1, systemcontracts.SystemContractCodeLookup[ex.chainConfig.ChainName]) + defer stateReader.Dispose() + + ibs := state.New(stateReader) + blockCtx := core.NewEVMBlockContext(header, core.GetHashFn(header, ex.getHeader), ex.engine, nil /* author */) + evm := vm.NewEVM(blockCtx, evmtypes.TxContext{}, ibs, ex.chainConfig, vm.Config{NoBaseFee: true}) + + return ex.prober.Probe(ex.ctx, evm, header, ex.chainConfig, ibs, blockNumber, addr, k, v) +} diff --git a/eth/stagedsync/ots_stg_executor_erc20_721_holder.go b/eth/stagedsync/ots_stg_executor_erc20_721_holder.go new file mode 100644 index 00000000000..080ff3e5e29 --- /dev/null +++ b/eth/stagedsync/ots_stg_executor_erc20_721_holder.go @@ -0,0 +1,30 @@ +package stagedsync + +import ( + "context" + "time" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func ERC20And721HolderIndexerExecutor(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) { + analyzer, err := NewTransferLogAnalyzer() + if err != nil { + return startBlock, err + } + + aggrHandler := NewMultiIndexerHandler[TransferAnalysisResult]( + NewTransferLogHolderHandler(tmpDir, s, false, kv.OtsERC20Holdings, logger), + NewTransferLogHolderHandler(tmpDir, s, true, kv.OtsERC721Holdings, logger), + ) + defer aggrHandler.Close() + + if startBlock == 0 && isInternalTx { + return runConcurrentLogIndexerExecutor[TransferAnalysisResult](db, tx, blockReader, startBlock, endBlock, isShortInterval, logEvery, ctx, s, analyzer, aggrHandler) + } + return runIncrementalLogIndexerExecutor[TransferAnalysisResult](db, tx, blockReader, startBlock, endBlock, isShortInterval, logEvery, ctx, s, analyzer, aggrHandler) +} diff --git a/eth/stagedsync/ots_stg_executor_erc20_721_transfer.go b/eth/stagedsync/ots_stg_executor_erc20_721_transfer.go new file mode 100644 index 00000000000..0dcaafac329 --- /dev/null +++ b/eth/stagedsync/ots_stg_executor_erc20_721_transfer.go @@ -0,0 +1,189 @@ +package stagedsync + +import ( + "bytes" + "context" + "time" + + lru "github.com/hashicorp/golang-lru" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func ERC20And721TransferIndexerExecutor(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) { + analyzer, err := NewTransferLogAnalyzer() + if err != nil { + return startBlock, err + } + + aggrHandler := NewMultiIndexerHandler[TransferAnalysisResult]( + NewTransferLogIndexerHandler(tmpDir, s, false, kv.OtsERC20TransferIndex, kv.OtsERC20TransferCounter, logger), + NewTransferLogIndexerHandler(tmpDir, s, true, kv.OtsERC721TransferIndex, kv.OtsERC721TransferCounter, logger), + ) + defer aggrHandler.Close() + + if startBlock == 0 && isInternalTx { + return runConcurrentLogIndexerExecutor[TransferAnalysisResult](db, tx, blockReader, startBlock, endBlock, isShortInterval, logEvery, ctx, s, analyzer, aggrHandler) + } + return runIncrementalLogIndexerExecutor[TransferAnalysisResult](db, tx, blockReader, startBlock, endBlock, isShortInterval, logEvery, ctx, s, analyzer, aggrHandler) +} + +// Topic hash for Transfer(address,address,uint256) event +var TRANSFER_TOPIC = hexutil.MustDecode("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + +type TransferAnalysisResult struct { + // ERC20 == false, ERC721 == true + nft bool + + // token address == log emitter address + token common.Address + + // topic1 + from common.Address + + // topic2 + to common.Address +} + +func (r *TransferAnalysisResult) Unwind(tx kv.RwTx, nft bool, indexer LogIndexerUnwinder, ethTx uint64) error { + if r.nft != nft { + return nil + } + + if err := indexer.UnwindAddress(tx, r.from, ethTx); err != nil { + return err + } + if err := indexer.UnwindAddress(tx, r.to, ethTx); err != nil { + return err + } + + return nil +} + +func (r *TransferAnalysisResult) UnwindHolding(tx kv.RwTx, nft bool, indexer LogIndexerUnwinder, ethTx uint64) error { + if r.nft != nft { + return nil + } + + if err := indexer.UnwindAddressHolding(tx, r.from, r.token, ethTx); err != nil { + return err + } + if err := indexer.UnwindAddressHolding(tx, r.to, r.token, ethTx); err != nil { + return err + } + + return nil +} + +// This is an implementation of LogAnalyzer that detects Transfer events. +type TransferLogAnalyzer struct { + // Caches positive/negative checks of address -> token? avoiding repeatedly DB checks + // for popular tokens. + erc20Cache *lru.ARCCache + + // Caches positive/negative checks of address -> token? avoiding repeatedly DB checks + // for popular tokens. + erc721Cache *lru.ARCCache +} + +func NewTransferLogAnalyzer() (*TransferLogAnalyzer, error) { + isERC20, err := lru.NewARC(1_000_000) + if err != nil { + return nil, err + } + + isERC721, err := lru.NewARC(1_000_000) + if err != nil { + return nil, err + } + + return &TransferLogAnalyzer{isERC20, isERC721}, nil +} + +// Checks if a log entry is a standard Transfer event. +// +// If so, the returned payload says if it is an ERC20 or ERC721 (both uses the same topic0), +// the from and to addresses. +func (a *TransferLogAnalyzer) Inspect(tx kv.Tx, l *types.Log) (*TransferAnalysisResult, error) { + // Transfer logs have 3 topics (ERC20: topic + from + to) or + // 4 (ERC721: topic + from + to + id) + if len(l.Topics) < 3 { + return nil, nil + } + + // Topic0 must match the standard Transfer topic sig + if !bytes.Equal(TRANSFER_TOPIC, l.Topics[0].Bytes()) { + return nil, nil + } + + // It is a Transfer(address, address) + tokenAddr := l.Address.Bytes() + + // Check token types + isERC20, isERC721, err := a.checkTokenType(tx, tokenAddr) + if err != nil { + return nil, err + } + if !isERC20 && !isERC721 { + return nil, nil + } + if isERC20 && isERC721 { + // Faulty token which identifies itself as both ERC20 and ERC721 + // log.Info("XXXXX BOTH", "tokenAddr", hexutility.Encode(tokenAddr)) + } + + // Confirmed that tokenAddr IS an ERC20 or ERC721 + fromAddr := common.BytesToAddress(l.Topics[1].Bytes()[length.Hash-length.Addr:]) + toAddr := common.BytesToAddress(l.Topics[2].Bytes()[length.Hash-length.Addr:]) + return &TransferAnalysisResult{isERC721, l.Address, fromAddr, toAddr}, nil +} + +func (a *TransferLogAnalyzer) checkTokenType(tx kv.Tx, tokenAddr []byte) (isERC20, isERC721 bool, err error) { + // Check caches + cachedERC20, okERC20 := a.erc20Cache.Get(string(tokenAddr)) + if okERC20 { + isERC20 = cachedERC20.(bool) + } + cachedERC721, okERC721 := a.erc721Cache.Get(string(tokenAddr)) + if okERC721 { + isERC721 = cachedERC721.(bool) + } + if okERC20 && okERC721 { + return isERC20, isERC721, nil + } + + // no entry == addr is not expect token type + attr, err := tx.GetOne(kv.OtsAddrAttributes, tokenAddr) + if err != nil { + return false, false, err + } + if attr == nil { + a.erc20Cache.Add(string(tokenAddr), false) + a.erc721Cache.Add(string(tokenAddr), false) + return false, false, nil + } + + // decode flag + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + + if _, err := bm.ReadFrom(bytes.NewReader(attr)); err != nil { + return false, false, err + } + + isERC20 = bm.Contains(kv.ADDR_ATTR_ERC20) + a.erc20Cache.Add(string(tokenAddr), isERC20) + + isERC721 = bm.Contains(kv.ADDR_ATTR_ERC721) + a.erc721Cache.Add(string(tokenAddr), isERC721) + + return isERC20, isERC721, nil +} diff --git a/eth/stagedsync/ots_stg_executor_withdrawals_indexer.go b/eth/stagedsync/ots_stg_executor_withdrawals_indexer.go new file mode 100644 index 00000000000..cd080fb90eb --- /dev/null +++ b/eth/stagedsync/ots_stg_executor_withdrawals_indexer.go @@ -0,0 +1,73 @@ +package stagedsync + +import ( + "context" + "time" + + "github.com/RoaringBitmap/roaring/roaring64" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/etl" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/consensus" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func WithdrawalsExecutor(ctx context.Context, db kv.RoDB, tx kv.RwTx, isInternalTx bool, tmpDir string, chainConfig *chain.Config, blockReader services.FullBlockReader, engine consensus.Engine, startBlock, endBlock uint64, isShortInterval bool, logEvery *time.Ticker, s *StageState, logger log.Logger) (uint64, error) { + withdrawalHandler, err := NewWithdrawalsIndexerHandler(tx, tmpDir, s, logger) + if err != nil { + return startBlock, err + } + defer withdrawalHandler.Close() + + return runIncrementalBodyIndexerExecutor(db, tx, blockReader, startBlock, endBlock, isShortInterval, logEvery, ctx, s, withdrawalHandler) +} + +// Implements BodyIndexerHandler interface in order to index block withdrawals from CL +type WithdrawalsIndexerHandler struct { + IndexHandler + withdrawalIdx2Block kv.RwCursor +} + +func NewWithdrawalsIndexerHandler(tx kv.RwTx, tmpDir string, s *StageState, logger log.Logger) (BodyIndexerHandler, error) { + collector := etl.NewCollector(s.LogPrefix(), tmpDir, etl.NewSortableBuffer(etl.BufferOptimalSize), logger) + bitmaps := map[string]*roaring64.Bitmap{} + withdrawalIdx2Block, err := tx.RwCursor(kv.OtsWithdrawalIdx2Block) + if err != nil { + return nil, err + } + + return &WithdrawalsIndexerHandler{ + &StandardIndexHandler{kv.OtsWithdrawalsIndex, kv.OtsWithdrawalsCounter, collector, bitmaps}, + withdrawalIdx2Block, + }, nil +} + +// Index all withdrawals from a block body; +// withdrawal address -> withdrawal index (NOT blockNum!!!) +func (h *WithdrawalsIndexerHandler) HandleMatch(blockNum uint64, body *types.Body) error { + withdrawals := body.Withdrawals + if len(withdrawals) == 0 { + return nil + } + last := withdrawals[len(withdrawals)-1] + + k := hexutility.EncodeTs(last.Index) + v := hexutility.EncodeTs(blockNum) + if err := h.withdrawalIdx2Block.Put(k, v); err != nil { + return err + } + + for _, w := range withdrawals { + h.TouchIndex(w.Address, w.Index) + } + + return nil +} + +func (h *WithdrawalsIndexerHandler) Close() { + h.IndexHandler.Close() + h.withdrawalIdx2Block.Close() +} diff --git a/eth/stagedsync/ots_unwinder_blocks_rewarded_runner.go b/eth/stagedsync/ots_unwinder_blocks_rewarded_runner.go new file mode 100644 index 00000000000..0bc6cfd7bab --- /dev/null +++ b/eth/stagedsync/ots_unwinder_blocks_rewarded_runner.go @@ -0,0 +1,41 @@ +package stagedsync + +import ( + "context" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func RunBlocksRewardedBlockUnwind(ctx context.Context, tx kv.RwTx, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker, u *UnwindState, unwinder IndexUnwinder) error { + // The unwind interval is ]u.UnwindPoint, EOF] + startBlock := u.UnwindPoint + 1 + + for blockNum := startBlock; blockNum <= u.CurrentBlockNumber; blockNum++ { + hash, err := blockReader.CanonicalHash(ctx, tx, blockNum) + if err != nil { + return err + } + header, err := blockReader.HeaderByHash(ctx, tx, hash) + if err != nil { + return err + } + + if err := unwinder.UnwindAddress(tx, header.Coinbase, header.Number.Uint64()); err != nil { + return err + } + + select { + default: + case <-ctx.Done(): + return common.ErrStopped + case <-logEvery.C: + log.Info("Unwinding blocks rewarded indexer", "blockNum", blockNum) + } + } + + return nil +} diff --git a/eth/stagedsync/ots_unwinder_executor_block_indexer.go b/eth/stagedsync/ots_unwinder_executor_block_indexer.go new file mode 100644 index 00000000000..e4dba095117 --- /dev/null +++ b/eth/stagedsync/ots_unwinder_executor_block_indexer.go @@ -0,0 +1,69 @@ +package stagedsync + +import ( + "context" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/turbo/services" +) + +type BlockUnwinderRunner func(ctx context.Context, tx kv.RwTx, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker, u *UnwindState, unwinder IndexUnwinder) error + +func NewGenericBlockIndexerUnwinder(bucket, counterBucket string, unwinderRunner BlockUnwinderRunner) UnwindExecutor { + return func(ctx context.Context, tx kv.RwTx, u *UnwindState, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker) error { + unwinder, err := newBlockIndexerUnwinder(tx, bucket, counterBucket) + if err != nil { + return err + } + defer unwinder.Dispose() + + return unwinderRunner(ctx, tx, blockReader, isShortInterval, logEvery, u, unwinder) + } +} + +type BlockIndexerIndexerUnwinder struct { + indexBucket string + counterBucket string + target kv.RwCursor + targetDel kv.RwCursor + counter kv.RwCursorDupSort +} + +func newBlockIndexerUnwinder(tx kv.RwTx, indexBucket, counterBucket string) (*BlockIndexerIndexerUnwinder, error) { + target, err := tx.RwCursor(indexBucket) + if err != nil { + return nil, err + } + + targetDel, err := tx.RwCursor(indexBucket) + if err != nil { + return nil, err + } + + counter, err := tx.RwCursorDupSort(counterBucket) + if err != nil { + return nil, err + } + + return &BlockIndexerIndexerUnwinder{ + indexBucket, + counterBucket, + target, + targetDel, + counter, + }, nil +} + +func (u *BlockIndexerIndexerUnwinder) UnwindAddress(tx kv.RwTx, addr common.Address, ethTx uint64) error { + return unwindAddress(tx, u.target, u.targetDel, u.counter, u.indexBucket, u.counterBucket, addr, ethTx) +} + +func (u *BlockIndexerIndexerUnwinder) Dispose() error { + u.target.Close() + u.targetDel.Close() + u.counter.Close() + + return nil +} diff --git a/eth/stagedsync/ots_unwinder_executor_log_holdings.go b/eth/stagedsync/ots_unwinder_executor_log_holdings.go new file mode 100644 index 00000000000..3d42e7c2058 --- /dev/null +++ b/eth/stagedsync/ots_unwinder_executor_log_holdings.go @@ -0,0 +1,110 @@ +package stagedsync + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/turbo/services" +) + +func NewGenericLogHoldingsUnwinder() UnwindExecutor { + return func(ctx context.Context, tx kv.RwTx, u *UnwindState, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker) error { + erc20Unwinder, err := NewTransferLogHoldingsUnwinder(tx, kv.OtsERC20Holdings, false) + if err != nil { + return err + } + defer erc20Unwinder.Dispose() + + erc721Unwinder, err := NewTransferLogHoldingsUnwinder(tx, kv.OtsERC721Holdings, true) + if err != nil { + return err + } + defer erc721Unwinder.Dispose() + + return runLogUnwind(ctx, tx, blockReader, isShortInterval, logEvery, u, TRANSFER_TOPIC, []UnwindHandler{erc20Unwinder, erc721Unwinder}) + } +} + +type TransferLogHoldingsUnwinder struct { + indexBucket string + isNFT bool + target kv.RwCursorDupSort + targetDel kv.RwCursorDupSort +} + +func NewTransferLogHoldingsUnwinder(tx kv.RwTx, indexBucket string, isNFT bool) (*TransferLogHoldingsUnwinder, error) { + target, err := tx.RwCursorDupSort(indexBucket) + if err != nil { + return nil, err + } + + targetDel, err := tx.RwCursorDupSort(indexBucket) + if err != nil { + return nil, err + } + + return &TransferLogHoldingsUnwinder{ + indexBucket, + isNFT, + target, + targetDel, + }, nil +} + +func (u *TransferLogHoldingsUnwinder) Dispose() error { + u.target.Close() + u.targetDel.Close() + + return nil +} + +func (u *TransferLogHoldingsUnwinder) Unwind(tx kv.RwTx, results []*TransferAnalysisResult, ethTx uint64) error { + for _, r := range results { + if err := r.UnwindHolding(tx, u.isNFT, u, ethTx); err != nil { + return err + } + } + + return nil +} + +func (u *TransferLogHoldingsUnwinder) UnwindAddress(tx kv.RwTx, addr common.Address, ethTx uint64) error { + return fmt.Errorf("NOT IMPLEMENTED") +} + +func (u *TransferLogHoldingsUnwinder) UnwindAddressHolding(tx kv.RwTx, addr, token common.Address, ethTx uint64) error { + k := addr.Bytes() + v, err := u.target.SeekBothRange(k, token.Bytes()) + if err != nil { + return err + } + if k == nil { + return nil + } + if !bytes.HasPrefix(v, token.Bytes()) { + return nil + } + existingEthTx := binary.BigEndian.Uint64(v[length.Addr:]) + + // ignore touches after the first recognized holding occurrence + if ethTx > existingEthTx { + return nil + } + + // touches before the first recognized holding occurrence means DB corruption + if ethTx < existingEthTx { + return fmt.Errorf("db possibly corrupted: trying to unwind bucket=%s holder=%s token=%s ethTx=%d existingEthTx=%d", u.indexBucket, addr, token, ethTx, existingEthTx) + } + + if err := u.targetDel.DeleteExact(k, v); err != nil { + return err + } + + return nil +} diff --git a/eth/stagedsync/ots_unwinder_executor_log_indexer.go b/eth/stagedsync/ots_unwinder_executor_log_indexer.go new file mode 100644 index 00000000000..e3b619f28d5 --- /dev/null +++ b/eth/stagedsync/ots_unwinder_executor_log_indexer.go @@ -0,0 +1,175 @@ +package stagedsync + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func NewGenericLogIndexerUnwinder() UnwindExecutor { + return func(ctx context.Context, tx kv.RwTx, u *UnwindState, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker) error { + erc20Unwinder, err := NewTransferLogIndexerUnwinder(tx, kv.OtsERC20TransferIndex, kv.OtsERC20TransferCounter, false) + if err != nil { + return err + } + defer erc20Unwinder.Dispose() + + erc721Unwinder, err := NewTransferLogIndexerUnwinder(tx, kv.OtsERC721TransferIndex, kv.OtsERC721TransferCounter, true) + if err != nil { + return err + } + defer erc721Unwinder.Dispose() + + return runLogUnwind(ctx, tx, blockReader, isShortInterval, logEvery, u, TRANSFER_TOPIC, []UnwindHandler{erc20Unwinder, erc721Unwinder}) + } +} + +type LogIndexerUnwinder interface { + UnwindAddress(tx kv.RwTx, addr common.Address, ethTx uint64) error + UnwindAddressHolding(tx kv.RwTx, addr, token common.Address, ethTx uint64) error + Dispose() error +} + +type UnwindHandler interface { + Unwind(tx kv.RwTx, results []*TransferAnalysisResult, ethTx uint64) error +} + +type TransferLogIndexerUnwinder struct { + indexBucket string + counterBucket string + isNFT bool + target kv.RwCursor + targetDel kv.RwCursor + counter kv.RwCursorDupSort +} + +func NewTransferLogIndexerUnwinder(tx kv.RwTx, indexBucket, counterBucket string, isNFT bool) (*TransferLogIndexerUnwinder, error) { + target, err := tx.RwCursor(indexBucket) + if err != nil { + return nil, err + } + + targetDel, err := tx.RwCursor(indexBucket) + if err != nil { + return nil, err + } + + counter, err := tx.RwCursorDupSort(counterBucket) + if err != nil { + return nil, err + } + + return &TransferLogIndexerUnwinder{ + indexBucket, + counterBucket, + isNFT, + target, + targetDel, + counter, + }, nil +} + +func (u *TransferLogIndexerUnwinder) Dispose() error { + u.target.Close() + u.targetDel.Close() + u.counter.Close() + + return nil +} + +func runLogUnwind(ctx context.Context, tx kv.RwTx, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker, u *UnwindState, topic []byte, unwinders []UnwindHandler) error { + analyzer, err := NewTransferLogAnalyzer() + if err != nil { + return err + } + + logs, err := tx.Cursor(kv.Log) + if err != nil { + return err + } + defer logs.Close() + + // The unwind interval is ]u.UnwindPoint, EOF] + startBlock := u.UnwindPoint + 1 + + // Traverse blocks logs [startBlock, EOF], determine txs that should've matched the criteria, + // their logs, and their addresses. + blocks, err := newBlockBitmapFromTopic(tx, startBlock, u.CurrentBlockNumber, topic) + if err != nil { + return err + } + defer bitmapdb.ReturnToPool(blocks) + + for it := blocks.Iterator(); it.HasNext(); { + blockNum := uint64(it.Next()) + + // Avoid recalculating txid from the block basetxid for each match + baseTxId, err := blockReader.BaseTxIdForBlock(ctx, tx, blockNum) + if err != nil { + return err + } + + // Inspect each block's tx logs + logPrefix := hexutility.EncodeTs(blockNum) + k, v, err := logs.Seek(logPrefix) + if err != nil { + return err + } + for k != nil && bytes.HasPrefix(k, logPrefix) { + txLogs := newTxLogsFromRaw[TransferAnalysisResult](blockNum, baseTxId, k, v) + results, err := AnalyzeLogs[TransferAnalysisResult](tx, analyzer, txLogs.rawLogs) + if err != nil { + return err + } + + if len(results) > 0 { + for _, unwinder := range unwinders { + if err := unwinder.Unwind(tx, results, txLogs.ethTx); err != nil { + return err + } + } + } + + select { + default: + case <-ctx.Done(): + return common.ErrStopped + case <-logEvery.C: + log.Info("Unwinding log indexer", "blockNum", blockNum) + } + + k, v, err = logs.Next() + if err != nil { + return err + } + } + } + + return nil +} + +func (u *TransferLogIndexerUnwinder) Unwind(tx kv.RwTx, results []*TransferAnalysisResult, ethTx uint64) error { + for _, r := range results { + if err := r.Unwind(tx, u.isNFT, u, ethTx); err != nil { + return err + } + } + + return nil +} + +func (u *TransferLogIndexerUnwinder) UnwindAddressHolding(tx kv.RwTx, addr, token common.Address, ethTx uint64) error { + return fmt.Errorf("NOT IMPLEMENTED; SHOULDN'T BE CALLED") +} + +func (u *TransferLogIndexerUnwinder) UnwindAddress(tx kv.RwTx, addr common.Address, ethTx uint64) error { + return unwindAddress(tx, u.target, u.targetDel, u.counter, u.indexBucket, u.counterBucket, addr, ethTx) +} diff --git a/eth/stagedsync/ots_unwinder_withdrawals_runner.go b/eth/stagedsync/ots_unwinder_withdrawals_runner.go new file mode 100644 index 00000000000..1f89b00f6f3 --- /dev/null +++ b/eth/stagedsync/ots_unwinder_withdrawals_runner.go @@ -0,0 +1,92 @@ +package stagedsync + +import ( + "context" + "fmt" + "time" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/turbo/services" + "github.com/ledgerwatch/log/v3" +) + +func RunWithdrawalsBlockUnwind(ctx context.Context, tx kv.RwTx, blockReader services.FullBlockReader, isShortInterval bool, logEvery *time.Ticker, u *UnwindState, unwinder IndexUnwinder) error { + // The unwind interval is ]u.UnwindPoint, EOF] + startBlock := u.UnwindPoint + 1 + + idx2Block, err := tx.RwCursor(kv.OtsWithdrawalIdx2Block) + if err != nil { + return err + } + defer idx2Block.Close() + + // In order to unwind idx2Block, we need to find the max withdrawal ID from the unwind point + // block or less + blockNum := u.UnwindPoint + found := false + withdrawalId := uint64(0) + for blockNum > 0 { + hash, err := blockReader.CanonicalHash(ctx, tx, blockNum) + if err != nil { + return err + } + body, _, err := blockReader.Body(ctx, tx, hash, blockNum) + if err != nil { + return err + } + + withdrawalsAmount := len(body.Withdrawals) + if withdrawalsAmount > 0 { + found = true + lastWithdrawal := body.Withdrawals[withdrawalsAmount-1] + withdrawalId = lastWithdrawal.Index + break + } + + blockNum-- + } + + // Unwind idx2Block + if found { + unwoundToIndex, err := unwindUint64KeyBasedTable(idx2Block, withdrawalId) + if err != nil { + return err + } + + // withdrawal ID MUST exist in idx2Block, otherwise it is a DB inconsistency + if unwoundToIndex != withdrawalId { + return fmt.Errorf("couldn't find bucket=%s k=%v to unwind; probably DB corruption", kv.OtsWithdrawalIdx2Block, withdrawalId) + } + } + + for blockNum := startBlock; blockNum <= u.CurrentBlockNumber; blockNum++ { + hash, err := blockReader.CanonicalHash(ctx, tx, blockNum) + if err != nil { + return err + } + body, _, err := blockReader.Body(ctx, tx, hash, blockNum) + if err != nil { + return err + } + + if len(body.Withdrawals) == 0 { + continue + } + for _, w := range body.Withdrawals { + if err := unwinder.UnwindAddress(tx, w.Address, w.Index); err != nil { + return err + } + } + + select { + default: + case <-ctx.Done(): + return common.ErrStopped + case <-logEvery.C: + log.Info("Unwinding withdrawals indexer", "blockNum", blockNum) + } + } + + return nil +} diff --git a/eth/stagedsync/otscontracts/IERC4626.json b/eth/stagedsync/otscontracts/IERC4626.json new file mode 100644 index 00000000000..e467fdf1ec0 --- /dev/null +++ b/eth/stagedsync/otscontracts/IERC4626.json @@ -0,0 +1,614 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "sender", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "assets", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "Deposit", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "from", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "sender", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "receiver", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "assets", + "type": "uint256" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "Withdraw", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "internalType": "address", + "name": "spender", + "type": "address" + } + ], + "name": "allowance", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "spender", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "approve", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "asset", + "outputs": [ + { + "internalType": "address", + "name": "assetTokenAddress", + "type": "address" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "account", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "convertToAssets", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "name": "convertToShares", + "outputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "decimals", + "outputs": [ + { + "internalType": "uint8", + "name": "", + "type": "uint8" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + }, + { + "internalType": "address", + "name": "receiver", + "type": "address" + } + ], + "name": "deposit", + "outputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "receiver", + "type": "address" + } + ], + "name": "maxDeposit", + "outputs": [ + { + "internalType": "uint256", + "name": "maxAssets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "receiver", + "type": "address" + } + ], + "name": "maxMint", + "outputs": [ + { + "internalType": "uint256", + "name": "maxShares", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "owner", + "type": "address" + } + ], + "name": "maxRedeem", + "outputs": [ + { + "internalType": "uint256", + "name": "maxShares", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "owner", + "type": "address" + } + ], + "name": "maxWithdraw", + "outputs": [ + { + "internalType": "uint256", + "name": "maxAssets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + }, + { + "internalType": "address", + "name": "receiver", + "type": "address" + } + ], + "name": "mint", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "name", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "name": "previewDeposit", + "outputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "previewMint", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "name": "previewRedeem", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "name": "previewWithdraw", + "outputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + }, + { + "internalType": "address", + "name": "receiver", + "type": "address" + }, + { + "internalType": "address", + "name": "owner", + "type": "address" + } + ], + "name": "redeem", + "outputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "symbol", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalAssets", + "outputs": [ + { + "internalType": "uint256", + "name": "totalManagedAssets", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "transfer", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "from", + "type": "address" + }, + { + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "transferFrom", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "uint256", + "name": "assets", + "type": "uint256" + }, + { + "internalType": "address", + "name": "receiver", + "type": "address" + }, + { + "internalType": "address", + "name": "owner", + "type": "address" + } + ], + "name": "withdraw", + "outputs": [ + { + "internalType": "uint256", + "name": "shares", + "type": "uint256" + } + ], + "stateMutability": "nonpayable", + "type": "function" + } +] \ No newline at end of file diff --git a/eth/stagedsync/otscontracts/erc165.json b/eth/stagedsync/otscontracts/erc165.json new file mode 100644 index 00000000000..bd6b6e2d755 --- /dev/null +++ b/eth/stagedsync/otscontracts/erc165.json @@ -0,0 +1,21 @@ +[ + { + "inputs": [ + { + "internalType": "bytes4", + "name": "interfaceId", + "type": "bytes4" + } + ], + "name": "supportsInterface", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "view", + "type": "function" + } +] \ No newline at end of file diff --git a/eth/stagedsync/otscontracts/erc20.json b/eth/stagedsync/otscontracts/erc20.json new file mode 100644 index 00000000000..1636031a03d --- /dev/null +++ b/eth/stagedsync/otscontracts/erc20.json @@ -0,0 +1,224 @@ +[ + { + "inputs": [], + "name": "name", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "symbol", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "decimals", + "outputs": [ + { + "internalType": "uint8", + "name": "", + "type": "uint8" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalSupply", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "account", + "type": "address" + } + ], + "name": "balanceOf", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "recipient", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "transfer", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "sender", + "type": "address" + }, + { + "internalType": "address", + "name": "recipient", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "transferFrom", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "spender", + "type": "address" + }, + { + "internalType": "uint256", + "name": "amount", + "type": "uint256" + } + ], + "name": "approve", + "outputs": [ + { + "internalType": "bool", + "name": "", + "type": "bool" + } + ], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "internalType": "address", + "name": "spender", + "type": "address" + } + ], + "name": "allowance", + "outputs": [ + { + "internalType": "uint256", + "name": "", + "type": "uint256" + } + ], + "stateMutability": "view", + "type": "function" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "from", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + } +] \ No newline at end of file diff --git a/eth/stagedsync/otscontracts/junk.json b/eth/stagedsync/otscontracts/junk.json new file mode 100644 index 00000000000..545e81a42c5 --- /dev/null +++ b/eth/stagedsync/otscontracts/junk.json @@ -0,0 +1,15 @@ +[ + { + "inputs": [], + "name": "junkjunkjunk", + "outputs": [ + { + "internalType": "string", + "name": "", + "type": "string" + } + ], + "stateMutability": "view", + "type": "function" + } +] \ No newline at end of file diff --git a/eth/stagedsync/otscontracts/otterscan_contracts.go b/eth/stagedsync/otscontracts/otterscan_contracts.go new file mode 100644 index 00000000000..688e654c6da --- /dev/null +++ b/eth/stagedsync/otscontracts/otterscan_contracts.go @@ -0,0 +1,17 @@ +package otscontracts + +import ( + _ "embed" +) + +//go:embed erc20.json +var ERC20 []byte + +//go:embed erc165.json +var ERC165 []byte + +//go:embed IERC4626.json +var IERC4626 []byte + +//go:embed junk.json +var Junk []byte diff --git a/eth/stagedsync/stages/stages.go b/eth/stagedsync/stages/stages.go index c6734f3923e..99a85e53663 100644 --- a/eth/stagedsync/stages/stages.go +++ b/eth/stagedsync/stages/stages.go @@ -58,6 +58,19 @@ var ( BeaconState SyncStage = "BeaconState" // Beacon blocks are sent to the state transition function BeaconIndexes SyncStage = "BeaconIndexes" // Fills up Beacon indexes + OtsContractIndexer SyncStage = "OtsContractIndexer" + OtsERC20Indexer SyncStage = "OtsERC20Indexer" + OtsERC165Indexer SyncStage = "OtsERC165Indexer" + OtsERC721Indexer SyncStage = "OtsERC721Indexer" + OtsERC1155Indexer SyncStage = "OtsERC1155Indexer" + OtsERC1167Indexer SyncStage = "OtsERC1167Indexer" + OtsERC4626Indexer SyncStage = "OtsERC4626Indexer" + + OtsERC20And721Holdings SyncStage = "OtsERC20And721Holdings" + OtsERC20And721Transfers SyncStage = "OtsERC20And721Transfers" + + OtsBlocksRewarded SyncStage = "OtsBlocksRewarded" + OtsWithdrawals SyncStage = "OtsWithdrawals" ) var AllStages = []SyncStage{ diff --git a/go.mod b/go.mod index 4aaa1848490..c4e7580c8ba 100644 --- a/go.mod +++ b/go.mod @@ -50,6 +50,7 @@ require ( github.com/google/gofuzz v1.2.0 github.com/gorilla/websocket v1.5.1 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 + github.com/hashicorp/golang-lru v0.5.1 github.com/hashicorp/golang-lru/arc/v2 v2.0.6 github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/holiman/uint256 v1.2.3 @@ -88,6 +89,7 @@ require ( github.com/valyala/fastjson v1.6.4 github.com/vektah/gqlparser/v2 v2.5.10 github.com/xsleonard/go-merkle v1.1.0 + go.uber.org/atomic v1.11.0 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 diff --git a/go.sum b/go.sum index 5484d069ca9..c98d0f2fa33 100644 --- a/go.sum +++ b/go.sum @@ -477,6 +477,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDa github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/arc/v2 v2.0.6 h1:4NU7uP5vSoK6TbaMj3NtY478TTAWLso/vL1gpNrInHg= github.com/hashicorp/golang-lru/arc/v2 v2.0.6/go.mod h1:cfdDIX05DWvYV6/shsxDfa/OVcRieOt+q4FnM8x+Xno= @@ -945,6 +946,7 @@ go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOl go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ= diff --git a/ots/indexer/counter.go b/ots/indexer/counter.go new file mode 100644 index 00000000000..5ea8b93dea1 --- /dev/null +++ b/ots/indexer/counter.go @@ -0,0 +1,30 @@ +package indexer + +import ( + "encoding/binary" + + "github.com/ledgerwatch/erigon-lib/common/length" +) + +func OptimizedCounterSerializer(count uint64) []byte { + v := make([]byte, 1) // 1 byte (counter - 1) [0, 255] + v[0] = byte(count - 1) + return v +} + +func RegularCounterSerializer(count uint64, chunk []byte) []byte { + // key == address + // value (dup) == accumulated counter uint64 + chunk uint64 + v := make([]byte, length.Counter+length.Chunk) + binary.BigEndian.PutUint64(v, count) + copy(v[length.Counter:], chunk) + return v +} + +func LastCounterSerializer(count uint64) []byte { + res := make([]byte, length.Counter+length.Chunk) + binary.BigEndian.PutUint64(res, count) + binary.BigEndian.PutUint64(res[length.Counter:], ^uint64(0)) + + return res +} diff --git a/ots/indexer/counter_test.go b/ots/indexer/counter_test.go new file mode 100644 index 00000000000..b98dfd01784 --- /dev/null +++ b/ots/indexer/counter_test.go @@ -0,0 +1,44 @@ +package indexer + +import ( + "bytes" + "testing" + + "github.com/ledgerwatch/erigon-lib/common/hexutility" +) + +func checkCounter(t *testing.T, result []byte, expected string) { + if !bytes.Equal(result, hexutility.MustDecodeHex(expected)) { + t.Errorf("got %s expected %s", hexutility.Encode(result), expected) + } +} + +func TestOptimizedCounterSerializerMin(t *testing.T) { + r := OptimizedCounterSerializer(1) + expected := "0x00" + checkCounter(t, r, expected) +} + +func TestOptimizedCounterSerializerMax(t *testing.T) { + r := OptimizedCounterSerializer(256) + expected := "0xff" + checkCounter(t, r, expected) +} + +func TestRegularCounterSerializer(t *testing.T) { + r := RegularCounterSerializer(257, hexutility.MustDecodeHex("0x1234567812345678")) + expected := "0x00000000000001011234567812345678" + checkCounter(t, r, expected) +} + +func TestLastCounterSerializerMin(t *testing.T) { + r := LastCounterSerializer(0) + expected := "0x0000000000000000ffffffffffffffff" + checkCounter(t, r, expected) +} + +func TestLastCounterSerializerMax(t *testing.T) { + r := LastCounterSerializer(^uint64(0)) + expected := "0xffffffffffffffffffffffffffffffff" + checkCounter(t, r, expected) +} diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index 45a39a2a963..fd8f1bcd733 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -162,6 +162,7 @@ var DefaultFlags = []cli.Flag{ &utils.SentinelPortFlag, &utils.OtsSearchMaxCapFlag, + &utils.OtsV2Flag, &utils.SilkwormExecutionFlag, &utils.SilkwormRpcDaemonFlag, diff --git a/turbo/jsonrpc/daemon.go b/turbo/jsonrpc/daemon.go index ca89bcd0809..319db101d54 100644 --- a/turbo/jsonrpc/daemon.go +++ b/turbo/jsonrpc/daemon.go @@ -50,6 +50,7 @@ func APIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, m } otsImpl := NewOtterscanAPI(base, db, cfg.OtsMaxPageSize) + ots2Impl := NewOtterscan2API(base, db) gqlImpl := NewGraphQLAPI(base, db) if cfg.GraphQLEnabled { @@ -149,6 +150,13 @@ func APIList(db kv.RoDB, eth rpchelper.ApiBackend, txPool txpool.TxpoolClient, m Service: OtterscanAPI(otsImpl), Version: "1.0", }) + case "ots2": + list = append(list, rpc.API{ + Namespace: "ots2", + Public: true, + Service: Otterscan2API(ots2Impl), + Version: "1.0", + }) case "clique": list = append(list, clique.NewCliqueAPI(db, engine, blockReader)) } diff --git a/turbo/jsonrpc/otterscan2_addr_attributes.go b/turbo/jsonrpc/otterscan2_addr_attributes.go new file mode 100644 index 00000000000..023531c438b --- /dev/null +++ b/turbo/jsonrpc/otterscan2_addr_attributes.go @@ -0,0 +1,49 @@ +package jsonrpc + +import ( + "bytes" + "context" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" +) + +type AddrAttributes struct { + ERC20 bool `json:"erc20,omitempty"` + ERC165 bool `json:"erc165,omitempty"` + ERC721 bool `json:"erc721,omitempty"` + ERC1155 bool `json:"erc1155,omitempty"` + ERC1167 bool `json:"erc1167,omitempty"` +} + +func (api *Otterscan2APIImpl) GetAddressAttributes(ctx context.Context, addr common.Address) (*AddrAttributes, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + v, err := tx.GetOne(kv.OtsAddrAttributes, addr.Bytes()) + if err != nil { + return nil, err + } + if v == nil { + return &AddrAttributes{}, nil + } + + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + if _, err := bm.ReadFrom(bytes.NewReader(v)); err != nil { + return nil, err + } + + attr := AddrAttributes{ + ERC20: bm.Contains(kv.ADDR_ATTR_ERC20), + ERC165: bm.Contains(kv.ADDR_ATTR_ERC165), + ERC721: bm.Contains(kv.ADDR_ATTR_ERC721), + ERC1155: bm.Contains(kv.ADDR_ATTR_ERC1155), + ERC1167: bm.Contains(kv.ADDR_ATTR_ERC1167), + } + return &attr, nil +} diff --git a/turbo/jsonrpc/otterscan2_all_contracts.go b/turbo/jsonrpc/otterscan2_all_contracts.go new file mode 100644 index 00000000000..a932163eec4 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_all_contracts.go @@ -0,0 +1,62 @@ +package jsonrpc + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" +) + +type ContractMatch struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` +} + +func (api *Otterscan2APIImpl) GetAllContractsList(ctx context.Context, idx, count uint64) (*ContractListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + res, err := api.genericMatchingList(ctx, tx, kv.OtsAllContracts, kv.OtsAllContractsCounter, idx, count) + if err != nil { + return nil, err + } + + extraData, err := api.newContractExtraData(ctx) + if err != nil { + return nil, err + } + + results, err := api.genericExtraData(ctx, tx, res, extraData) + if err != nil { + return nil, err + } + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, ToBlockSlice(res)) + if err != nil { + return nil, err + } + return &ContractListResult{ + BlocksSummary: blocksSummary, + Results: results, + }, nil +} + +func (api *Otterscan2APIImpl) newContractExtraData(ctx context.Context) (ExtraDataExtractor, error) { + return func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) { + return &ContractMatch{ + res.Block, + res.Address, + }, nil + }, nil +} + +func (api *Otterscan2APIImpl) GetAllContractsCount(ctx context.Context) (uint64, error) { + return api.genericMatchingCounter(ctx, kv.OtsAllContractsCounter) +} diff --git a/turbo/jsonrpc/otterscan2_api.go b/turbo/jsonrpc/otterscan2_api.go new file mode 100644 index 00000000000..6c542a9c80b --- /dev/null +++ b/turbo/jsonrpc/otterscan2_api.go @@ -0,0 +1,301 @@ +package jsonrpc + +import ( + "context" + "fmt" + "time" + + "github.com/holiman/uint256" + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/core/vm/evmtypes" + "github.com/ledgerwatch/erigon/rpc" + "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" + "github.com/ledgerwatch/erigon/turbo/rpchelper" + "github.com/ledgerwatch/log/v3" +) + +type Otterscan2API interface { + GetAllContractsList(ctx context.Context, idx, count uint64) (*ContractListResult, error) + GetAllContractsCount(ctx context.Context) (uint64, error) + GetERC20List(ctx context.Context, idx, count uint64) (*ContractListResult, error) + GetERC20Count(ctx context.Context) (uint64, error) + GetERC721List(ctx context.Context, idx, count uint64) (*ContractListResult, error) + GetERC721Count(ctx context.Context) (uint64, error) + GetERC1155List(ctx context.Context, idx, count uint64) (*ContractListResult, error) + GetERC1155Count(ctx context.Context) (uint64, error) + GetERC1167List(ctx context.Context, idx, count uint64) (*ContractListResult, error) + GetERC1167Count(ctx context.Context) (uint64, error) + GetERC4626List(ctx context.Context, idx, count uint64) (*ContractListResult, error) + GetERC4626Count(ctx context.Context) (uint64, error) + + GetERC1167Impl(ctx context.Context, addr common.Address) (common.Address, error) + + GetAddressAttributes(ctx context.Context, addr common.Address) (*AddrAttributes, error) + + GetERC20TransferList(ctx context.Context, addr common.Address, idx, count uint64) (*TransactionListResult, error) + GetERC20TransferCount(ctx context.Context, addr common.Address) (uint64, error) + GetERC721TransferList(ctx context.Context, addr common.Address, idx, count uint64) (*TransactionListResult, error) + GetERC721TransferCount(ctx context.Context, addr common.Address) (uint64, error) + GetERC20Holdings(ctx context.Context, addr common.Address) ([]*HoldingMatch, error) + GetERC721Holdings(ctx context.Context, addr common.Address) ([]*HoldingMatch, error) + + GetBlocksRewardedList(ctx context.Context, addr common.Address, idx, count uint64) (*BlocksRewardedListResult, error) + GetBlocksRewardedCount(ctx context.Context, addr common.Address) (uint64, error) + GetWithdrawalsList(ctx context.Context, addr common.Address, idx, count uint64) (*WithdrawalsListResult, error) + GetWithdrawalsCount(ctx context.Context, addr common.Address) (uint64, error) + + TransferIntegrityChecker(ctx context.Context) error + HoldingsIntegrityChecker(ctx context.Context) error +} + +type Otterscan2APIImpl struct { + *BaseAPI + db kv.RoDB +} + +func NewOtterscan2API(base *BaseAPI, db kv.RoDB) *Otterscan2APIImpl { + return &Otterscan2APIImpl{ + BaseAPI: base, + db: db, + } +} + +// Max results that can be requested by genericMatchingList callers to avoid node DoS +const MAX_MATCH_COUNT = uint64(500) + +// TODO: replace by BlockSummary2 +type BlockSummary struct { + Block hexutil.Uint64 `json:"blockNumber"` + Time uint64 `json:"timestamp"` +} + +type BlockSummary2 struct { + Block hexutil.Uint64 `json:"blockNumber"` + Time uint64 `json:"timestamp"` + internalIssuance + TotalFees hexutil.Uint64 `json:"totalFees"` +} + +type AddrMatch struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` +} + +func ToBlockSlice(addrMatches []AddrMatch) []hexutil.Uint64 { + res := make([]hexutil.Uint64, 0, len(addrMatches)) + for _, m := range addrMatches { + res = append(res, *m.Block) + } + return res +} + +func (api *Otterscan2APIImpl) newBlocksSummaryFromResults(ctx context.Context, tx kv.Tx, res []hexutil.Uint64) (map[hexutil.Uint64]*BlockSummary, error) { + ret := make(map[hexutil.Uint64]*BlockSummary, 0) + + for _, m := range res { + if _, ok := ret[m]; ok { + continue + } + + header, err := api._blockReader.HeaderByNumber(ctx, tx, uint64(m)) + if err != nil { + return nil, err + } + + ret[m] = &BlockSummary{m, header.Time} + } + + return ret, nil +} + +func (api *Otterscan2APIImpl) newBlocksSummary2FromResults(ctx context.Context, tx kv.Tx, res []hexutil.Uint64) (map[hexutil.Uint64]*BlockSummary2, error) { + ret := make(map[hexutil.Uint64]*BlockSummary2, 0) + + for _, m := range res { + if _, ok := ret[m]; ok { + continue + } + + number := rpc.BlockNumber(m) + b, senders, err := api.getBlockWithSenders(ctx, number, tx) + if err != nil { + return nil, err + } + if b == nil { + return nil, nil + } + details, err := api.getBlockDetailsImpl(ctx, tx, b, number, senders) + if err != nil { + return nil, err + } + + ret[m] = details + } + + return ret, nil +} + +// Given an address search match, extract some extra data by running EVM against it +type ExtraDataExtractor func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) + +func (api *Otterscan2APIImpl) genericExtraData(ctx context.Context, tx kv.Tx, res []AddrMatch, extraData ExtraDataExtractor) ([]interface{}, error) { + newRes := make([]interface{}, 0, len(res)) + + ticker := time.NewTicker(20 * time.Second) + defer ticker.Stop() + + var evm *vm.EVM + stateReader := state.NewPlainStateReader(tx) + // var ibs *state.IntraBlockState + ibs := state.New(stateReader) + prevBlock := uint64(0) + + blockReader := api._blockReader + chainConfig, err := api.chainConfig(tx) + if err != nil { + return nil, err + } + getHeader := func(hash common.Hash, number uint64) *types.Header { + h, e := blockReader.Header(ctx, tx, hash, number) + if e != nil { + log.Error("getHeader error", "number", number, "hash", hash, "err", e) + } + return h + } + engine := api.engine() + + blockNumber, hash, _, err := rpchelper.GetCanonicalBlockNumber(rpc.BlockNumberOrHashWithNumber(rpc.LatestExecutedBlockNumber), tx, nil) + if err != nil { + return nil, err + } + block, err := api.blockWithSenders(tx, hash, blockNumber) + if err != nil { + return nil, err + } + if block == nil { + return nil, nil + } + header := block.HeaderNoCopy() + + for _, r := range res { + // TODO: this is failing when block is the tip, hence block + 1 header can't be found! + ///////////////////// + originalBlockNum := uint64(*r.Block + 1) + + // var header *types.Header + // header, err = blockReader.HeaderByNumber(ctx, tx, originalBlockNum) + // if err != nil { + // return nil, err + // } + if header == nil { + return nil, fmt.Errorf("couldn't find header for block %d", originalBlockNum) + } + + if stateReader == nil { + // stateReader = state.NewPlainStateReader(tx) + // stateReader = state.NewPlainState(tx, originalBlockNum+1, systemcontracts.SystemContractCodeLookup[chainConfig.ChainName]) + // ibs = state.New(stateReader) + } else if originalBlockNum != prevBlock { + // stateReader.SetBlockNr(originalBlockNum + 1) + // ibs.Reset() + } + + if evm == nil { + blockCtx := core.NewEVMBlockContext(header, core.GetHashFn(header, getHeader), engine, nil /* author */) + evm = vm.NewEVM(blockCtx, evmtypes.TxContext{}, ibs, chainConfig, vm.Config{NoBaseFee: true}) + } else { + if originalBlockNum != prevBlock { + // reset block + blockCtx := core.NewEVMBlockContext(header, core.GetHashFn(header, getHeader), engine, nil /* author */) + evm.ResetBetweenBlocks(blockCtx, evmtypes.TxContext{}, ibs, vm.Config{NoBaseFee: true}, chainConfig.Rules(blockCtx.BlockNumber, blockCtx.Time)) + } + } + prevBlock = originalBlockNum + ///////////////////// + + addr := r.Address + ibs.Reset() + extra, err := extraData(tx, &r, *addr, evm, header, chainConfig, ibs, stateReader) + if err != nil { + return nil, err + } + newRes = append(newRes, extra) + + select { + default: + case <-ctx.Done(): + return nil, common.ErrStopped + } + } + + return newRes, nil +} + +func decodeReturnData(ctx context.Context, addr *common.Address, data []byte, methodName string, header *types.Header, evm *vm.EVM, chainConfig *chain.Config, ibs *state.IntraBlockState, contractABI *abi.ABI) (interface{}, error) { + gas := hexutil.Uint64(header.GasLimit) + args := ethapi.CallArgs{ + To: addr, + Data: (*hexutility.Bytes)(&data), + Gas: &gas, + } + ret, err := probeContract(ctx, evm, header, chainConfig, ibs, args) + if err != nil { + // internal error + return nil, err + } + + if ret.Err != nil { + // ignore on purpose; i.e., out of gas signals error here + log.Warn(fmt.Sprintf("error while trying to unpack %s: %v", methodName, ret.Err)) + return nil, nil + } + + retVal, err := contractABI.Unpack(methodName, ret.ReturnData) + if err != nil { + // ignore on purpose; untrusted contract doesn't comply to expected ABI + log.Warn(fmt.Sprintf("error while trying to unpack %s: %v", methodName, err)) + return nil, nil + } + + return retVal[0], nil +} + +func probeContract(ctx context.Context, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, state *state.IntraBlockState, args ethapi.CallArgs) (*core.ExecutionResult, error) { + var baseFee *uint256.Int + if header != nil && header.BaseFee != nil { + var overflow bool + baseFee, overflow = uint256.FromBig(header.BaseFee) + if overflow { + return nil, fmt.Errorf("header.BaseFee uint256 overflow") + } + } + msg, err := args.ToMessage(0, baseFee) + if err != nil { + return nil, err + } + + txCtx := core.NewEVMTxContext(msg) + state.Reset() + evm.Reset(txCtx, state) + + gp := new(core.GasPool).AddGas(msg.Gas()) + result, err := core.ApplyMessage(evm, msg, gp, true /* refunds */, false /* gasBailout */) + if err != nil { + return nil, err + } + + // If the timer caused an abort, return an appropriate error message + if evm.Cancelled() { + return nil, fmt.Errorf("execution aborted (timeout = )") + } + return result, nil +} diff --git a/turbo/jsonrpc/otterscan2_api_contracts.go b/turbo/jsonrpc/otterscan2_api_contracts.go new file mode 100644 index 00000000000..f4699268d5f --- /dev/null +++ b/turbo/jsonrpc/otterscan2_api_contracts.go @@ -0,0 +1,128 @@ +package jsonrpc + +import ( + "context" + "encoding/binary" + "fmt" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type ContractListResult struct { + BlocksSummary map[hexutil.Uint64]*BlockSummary `json:"blocksSummary"` + Results []interface{} `json:"results"` +} + +func (api *Otterscan2APIImpl) genericMatchingList(ctx context.Context, tx kv.Tx, matchTable, counterTable string, idx, count uint64) ([]AddrMatch, error) { + if count > MAX_MATCH_COUNT { + return nil, fmt.Errorf("maximum allowed results: %v", MAX_MATCH_COUNT) + } + + if tx == nil { + var err error + tx, err = api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + } + + c, err := tx.Cursor(counterTable) + if err != nil { + return nil, err + } + defer c.Close() + + startIdx := idx + 1 + counterK, blockNumV, err := c.Seek(hexutility.EncodeTs(startIdx)) + if err != nil { + return nil, err + } + if counterK == nil { + return nil, nil + } + + prevCounterK, _, err := c.Prev() + if err != nil { + return nil, err + } + prevTotal := uint64(0) + if prevCounterK != nil { + prevTotal = binary.BigEndian.Uint64(prevCounterK) + } + + contracts, err := tx.CursorDupSort(matchTable) + if err != nil { + return nil, err + } + if err != nil { + return nil, err + } + kk, vv, err := contracts.SeekExact(blockNumV) + if err != nil { + return nil, err + } + if kk == nil { + // DB corrupted + return nil, fmt.Errorf("couldn't find exact block %v for counter index: %v, counter key: %v", binary.BigEndian.Uint64(blockNumV), startIdx, binary.BigEndian.Uint64(counterK)) + } + + // Position cursor at the first match + for i := uint64(0); i < startIdx-prevTotal-1; i++ { + kk, vv, err = contracts.NextDup() + if err != nil { + return nil, err + } + } + + matches := make([]AddrMatch, 0, count) + for i := uint64(0); i < count && kk != nil; i++ { + blockNum := hexutil.Uint64(binary.BigEndian.Uint64(kk)) + addr := common.BytesToAddress(vv) + matches = append(matches, AddrMatch{Block: &blockNum, Address: &addr}) + + kk, vv, err = contracts.NextDup() + if err != nil { + return nil, err + } + if kk == nil { + kk, vv, err = contracts.NextNoDup() + if err != nil { + return nil, err + } + if kk == nil { + break + } + } + } + + return matches, nil +} + +func (api *Otterscan2APIImpl) genericMatchingCounter(ctx context.Context, counterTable string) (uint64, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback() + + ct, err := tx.Cursor(counterTable) + if err != nil { + return 0, err + } + defer ct.Close() + + k, _, err := ct.Last() + if err != nil { + return 0, err + } + if k == nil { + return 0, nil + } + + counter := binary.BigEndian.Uint64(k) + return counter, nil +} diff --git a/turbo/jsonrpc/otterscan2_api_generic.go b/turbo/jsonrpc/otterscan2_api_generic.go new file mode 100644 index 00000000000..4baa436eb8c --- /dev/null +++ b/turbo/jsonrpc/otterscan2_api_generic.go @@ -0,0 +1,174 @@ +package jsonrpc + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" +) + +type SearchResultMaterializer[T any] interface { + Convert(ctx context.Context, tx kv.Tx, idx uint64) (*T, error) +} + +func genericResultList[T any](ctx context.Context, tx kv.Tx, addr common.Address, idx, count uint64, indexBucket, counterBucket string, srm SearchResultMaterializer[T]) ([]*T, error) { + chunks, err := tx.Cursor(indexBucket) + if err != nil { + return nil, err + } + defer chunks.Close() + + counter, err := tx.CursorDupSort(counterBucket) + if err != nil { + return nil, err + } + defer counter.Close() + + // Determine page [start, end] + startIdx := idx + 1 + + // Locate first chunk + counterFound, chunk, err := findNextCounter(counter, addr, startIdx) + if err != nil { + return nil, err + } + if counterFound == 0 { + return []*T{}, nil + } + + // Locate first chunk + chunkKey := make([]byte, length.Addr+length.Chunk) + copy(chunkKey, addr.Bytes()) + copy(chunkKey[length.Addr:], chunk) + k, v, err := chunks.SeekExact(chunkKey) + if err != nil { + return nil, err + } + if k == nil { + return nil, fmt.Errorf("db possibly corrupted, couldn't find chunkKey %s on bucket %s", hexutility.Encode(chunkKey), indexBucket) + } + + bm := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bm) + + for i := 0; i < len(v); i += 8 { + bm.Add(binary.BigEndian.Uint64(v[i : i+8])) + } + + // bitmap contains idxs [counterFound - bm.GetCardinality(), counterFound - 1] + startCounter := counterFound - bm.GetCardinality() + 1 + if startIdx > startCounter { + endRange, err := bm.Select(startIdx - startCounter - 1) + if err != nil { + return nil, err + } + bm.RemoveRange(bm.Minimum(), endRange+1) + } + + ret := make([]*T, 0) + it := bm.Iterator() + for c := uint64(0); c < count; c++ { + // Look at next chunk? + if !it.HasNext() { + k, v, err = chunks.Next() + if err != nil { + return nil, err + } + if !bytes.HasPrefix(k, addr.Bytes()) { + break + } + + bm.Clear() + for i := 0; i < len(v); i += 8 { + bm.Add(binary.BigEndian.Uint64(v[i : i+8])) + } + it = bm.Iterator() + } + + // Convert match ID to proper struct data (it maybe a tx, a withdraw, etc, determined by T) + idx := it.Next() + result, err := srm.Convert(ctx, tx, idx) + if err != nil { + return nil, err + } + ret = append(ret, result) + } + + return ret, nil +} + +// Given an index, locates the counter chunk which should contain the desired index (>= index) +func findNextCounter(counter kv.CursorDupSort, addr common.Address, idx uint64) (uint64, []byte, error) { + v, err := counter.SeekBothRange(addr.Bytes(), hexutility.EncodeTs(idx)) + if err != nil { + return 0, nil, err + } + + // No occurrences + if v == nil { + return 0, nil, nil + } + + // <= 256 matches-optimization + if len(v) == 1 { + c, err := counter.CountDuplicates() + if err != nil { + return 0, nil, err + } + if c != 1 { + return 0, nil, fmt.Errorf("db possibly corrupted, expected 1 duplicate, got %d", c) + } + return uint64(v[0]) + 1, []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, nil + } + + // Regular chunk + return binary.BigEndian.Uint64(v[:length.Ts]), v[length.Ts:], nil +} + +func (api *Otterscan2APIImpl) genericGetCount(ctx context.Context, addr common.Address, counterBucket string) (uint64, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback() + + counter, err := tx.CursorDupSort(counterBucket) + if err != nil { + return 0, err + } + defer counter.Close() + + k, _, err := counter.SeekExact(addr.Bytes()) + if err != nil { + return 0, err + } + if k == nil { + return 0, nil + } + + v, err := counter.LastDup() + if err != nil { + return 0, err + } + + // Check if it is in the <= 256 count optimization + if len(v) == 1 { + c, err := counter.CountDuplicates() + if err != nil { + return 0, err + } + if c != 1 { + return 0, fmt.Errorf("db possibly corrupted, expected 1 duplicate, got %d", c) + } + + return uint64(v[0]) + 1, nil + } + + return binary.BigEndian.Uint64(v[:8]), nil +} diff --git a/turbo/jsonrpc/otterscan2_api_transfers.go b/turbo/jsonrpc/otterscan2_api_transfers.go new file mode 100644 index 00000000000..4d52b1fcb77 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_api_transfers.go @@ -0,0 +1,199 @@ +package jsonrpc + +import ( + "context" + "fmt" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/rawdb" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/log/v3" +) + +type TransactionListResult struct { + BlocksSummary map[hexutil.Uint64]*BlockSummary `json:"blocksSummary"` + Results []*TransactionMatch `json:"results"` +} + +type TransactionMatch struct { + Hash common.Hash `json:"hash"` + Transaction *RPCTransaction `json:"transaction"` + Receipt map[string]interface{} `json:"receipt"` +} + +type transactionSearchResultMaterializer struct { + api *Otterscan2APIImpl +} + +func (m *transactionSearchResultMaterializer) Convert(ctx context.Context, tx kv.Tx, idx uint64) (*TransactionMatch, error) { + txn, err := m.api._blockReader.TxnByTxId(ctx, tx, idx) + if err != nil { + return nil, err + } + + blockNum, _, err := m.api.txnLookup(tx, txn.Hash()) + if err != nil { + return nil, err + } + block, err := m.api.blockByNumberWithSenders(tx, blockNum) + if err != nil { + return nil, err + } + if block == nil { + return nil, nil // not error, see https://github.com/ledgerwatch/erigon/issues/1645 + } + + receipt, err := m.api._getTransactionReceipt(ctx, tx, txn.Hash()) + if err != nil { + return nil, err + } + + result := &TransactionMatch{ + Hash: txn.Hash(), + Transaction: NewRPCTransaction(txn, block.Hash(), blockNum, 0, block.BaseFee()), + Receipt: receipt, + } + return result, nil +} + +// Implements a template method for API that expose an address-based search results, +// like ERC20 or ERC721 txs that contains transfers related to a certain address. +// +// Usually this method implements most part of the job, and caller methods just wrap +// it with corresponding DB tables. +// +// Semantics of corresponding parameters are the same in the caller methods, so it +// should be assumed this doc is the source of truth. +// +// The idx param is 0-based index of the first match that should be returned, considering +// the elements are numbered [0, numElem - 1]. +// +// The count param determines the maximum of how many results should be returned. It may +// return less than count elements if the table's last record is reached and there are +// no more results available. +// +// Those 2 params allow for a flexible way to build paginated results, i.e., you can get the +// 3rd page of results in a 25 element page by passing: idx == (3 - 1) * 25, count == 25. +// +// Most likely, for a search results when the matches are shown backwards in time, and pages +// are dynamically numbered backwards from the last search results, getting the 3rd page +// would require the client code to use: idx == (totalMatches - 3 * 25), count == 25; the +// search results should then be reversed in the UI. +func (api *Otterscan2APIImpl) genericTransferList(ctx context.Context, addr common.Address, idx, count uint64, indexBucket, counterBucket string) (*TransactionListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + var srm SearchResultMaterializer[TransactionMatch] = &transactionSearchResultMaterializer{api} + ret, err := genericResultList(ctx, tx, addr, idx, count, indexBucket, counterBucket, srm) + if err != nil { + return nil, err + } + + blocks := make([]hexutil.Uint64, 0, len(ret)) + for _, r := range ret { + blockNum, ok, err := api.txnLookup(tx, r.Hash) + if err != nil { + return nil, err + } + if !ok { + log.Warn("unexpected error, couldn't find tx", "hash", r.Hash) + } + blocks = append(blocks, hexutil.Uint64(blockNum)) + } + + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, blocks) + if err != nil { + return nil, err + } + return &TransactionListResult{ + BlocksSummary: blocksSummary, + Results: ret, + }, nil +} + +// copied from eth_receipts.go +func (api *Otterscan2APIImpl) _getTransactionReceipt(ctx context.Context, tx kv.Tx, txnHash common.Hash) (map[string]interface{}, error) { + var blockNum uint64 + var ok bool + + blockNum, ok, err := api.txnLookup(tx, txnHash) + if err != nil { + return nil, err + } + + cc, err := api.chainConfig(tx) + if err != nil { + return nil, err + } + + if !ok && cc.Bor == nil { + return nil, nil + } + + // if not ok and cc.Bor != nil then we might have a bor transaction. + // Note that Private API returns 0 if transaction is not found. + if !ok || blockNum == 0 { + blockNumPtr, err := rawdb.ReadBorTxLookupEntry(tx, txnHash) + if err != nil { + return nil, err + } + if blockNumPtr == nil { + return nil, nil + } + + blockNum = *blockNumPtr + } + + block, err := api.blockByNumberWithSenders(tx, blockNum) + if err != nil { + return nil, err + } + if block == nil { + return nil, nil // not error, see https://github.com/ledgerwatch/erigon/issues/1645 + } + + var txnIndex uint64 + var txn types.Transaction + for idx, transaction := range block.Transactions() { + if transaction.Hash() == txnHash { + txn = transaction + txnIndex = uint64(idx) + break + } + } + + var borTx types.Transaction + // if txn == nil { + // borTx = rawdb.ReadBorTransactionForBlock(tx, block) + // if borTx == nil { + // return nil, nil + // } + // } + + receipts, err := api.getReceipts(ctx, tx, cc, block, block.Body().SendersFromTxs()) + if err != nil { + return nil, fmt.Errorf("getReceipts error: %w", err) + } + + if txn == nil { + borReceipt, err := rawdb.ReadBorReceipt(tx, block.Hash(), blockNum, receipts) + if err != nil { + return nil, err + } + if borReceipt == nil { + return nil, nil + } + return marshalReceipt(borReceipt, borTx, cc, block.HeaderNoCopy(), txnHash, false), nil + } + + if len(receipts) <= int(txnIndex) { + return nil, fmt.Errorf("block has less receipts than expected: %d <= %d, block: %d", len(receipts), int(txnIndex), blockNum) + } + + return marshalReceipt(receipts[txnIndex], block.Transactions()[txnIndex], cc, block.HeaderNoCopy(), txnHash, true), nil +} diff --git a/turbo/jsonrpc/otterscan2_blocks_rewarded.go b/turbo/jsonrpc/otterscan2_blocks_rewarded.go new file mode 100644 index 00000000000..e1a1f493e7c --- /dev/null +++ b/turbo/jsonrpc/otterscan2_blocks_rewarded.go @@ -0,0 +1,126 @@ +package jsonrpc + +import ( + "context" + "fmt" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/rpc" + "github.com/ledgerwatch/erigon/turbo/rpchelper" + "github.com/ledgerwatch/erigon/turbo/services" +) + +type BlocksRewardedListResult struct { + BlocksSummary map[hexutil.Uint64]*BlockSummary2 `json:"blocksSummary"` + Results []*BlocksRewardedMatch `json:"results"` +} + +type BlocksRewardedMatch struct { + BlockNum hexutil.Uint64 `json:"blockNumber"` + // Amount hexutil.Uint64 `json:"amount"` +} + +type blocksRewardedSearchResultMaterializer struct { + blockReader services.FullBlockReader +} + +func NewBlocksRewardedSearchResultMaterializer(tx kv.Tx, blockReader services.FullBlockReader) (*blocksRewardedSearchResultMaterializer, error) { + return &blocksRewardedSearchResultMaterializer{blockReader}, nil +} + +func (w *blocksRewardedSearchResultMaterializer) Convert(ctx context.Context, tx kv.Tx, idx uint64) (*BlocksRewardedMatch, error) { + // hash, err := w.blockReader.CanonicalHash(ctx, tx, blockNum) + // if err != nil { + // return nil, err + // } + // TODO: replace by header + // body, _, err := w.blockReader.Body(ctx, tx, hash, blockNum) + // if err != nil { + // return nil, err + // } + + result := &BlocksRewardedMatch{ + BlockNum: hexutil.Uint64(idx), + } + return result, nil +} + +func (api *Otterscan2APIImpl) GetBlocksRewardedList(ctx context.Context, addr common.Address, idx, count uint64) (*BlocksRewardedListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + srm, err := NewBlocksRewardedSearchResultMaterializer(tx, api._blockReader) + if err != nil { + return nil, err + } + + ret, err := genericResultList(ctx, tx, addr, idx, count, kv.OtsBlocksRewardedIndex, kv.OtsBlocksRewardedCounter, (SearchResultMaterializer[BlocksRewardedMatch])(srm)) + if err != nil { + return nil, err + } + + blocks := make([]hexutil.Uint64, 0, len(ret)) + for _, r := range ret { + blocks = append(blocks, hexutil.Uint64(r.BlockNum)) + } + + blocksSummary, err := api.newBlocksSummary2FromResults(ctx, tx, blocks) + if err != nil { + return nil, err + } + return &BlocksRewardedListResult{ + BlocksSummary: blocksSummary, + Results: ret, + }, nil +} + +func (api *Otterscan2APIImpl) GetBlocksRewardedCount(ctx context.Context, addr common.Address) (uint64, error) { + return api.genericGetCount(ctx, addr, kv.OtsBlocksRewardedCounter) +} + +func (api *Otterscan2APIImpl) getBlockWithSenders(ctx context.Context, number rpc.BlockNumber, tx kv.Tx) (*types.Block, []common.Address, error) { + if number == rpc.PendingBlockNumber { + return api.pendingBlock(), nil, nil + } + + n, hash, _, err := rpchelper.GetBlockNumber(rpc.BlockNumberOrHashWithNumber(number), tx, api.filters) + if err != nil { + return nil, nil, err + } + + block, senders, err := api._blockReader.BlockWithSenders(ctx, tx, hash, n) + return block, senders, err +} + +func (api *Otterscan2APIImpl) getBlockDetailsImpl(ctx context.Context, tx kv.Tx, b *types.Block, number rpc.BlockNumber, senders []common.Address) (*BlockSummary2, error) { + var response BlockSummary2 + chainConfig, err := api.chainConfig(tx) + if err != nil { + return nil, err + } + + getIssuanceRes, err := delegateIssuance(tx, b, chainConfig) + if err != nil { + return nil, err + } + receipts, err := api.getReceipts(ctx, tx, chainConfig, b, senders) + if err != nil { + return nil, fmt.Errorf("getReceipts error: %v", err) + } + feesRes, err := delegateBlockFees(ctx, tx, b, senders, chainConfig, receipts) + if err != nil { + return nil, err + } + + response.Block = hexutil.Uint64(b.Number().Uint64()) + response.Time = b.Time() + response.internalIssuance = getIssuanceRes + response.TotalFees = hexutil.Uint64(feesRes) + return &response, nil +} diff --git a/turbo/jsonrpc/otterscan2_debug.go b/turbo/jsonrpc/otterscan2_debug.go new file mode 100644 index 00000000000..1cafb6c42be --- /dev/null +++ b/turbo/jsonrpc/otterscan2_debug.go @@ -0,0 +1,250 @@ +package jsonrpc + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/eth/stagedsync/stages" + "github.com/ledgerwatch/log/v3" +) + +func (api *Otterscan2APIImpl) TransferIntegrityChecker(ctx context.Context) error { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return err + } + defer tx.Rollback() + + if err := api.checkTransferIntegrity(tx, kv.OtsERC20TransferIndex, kv.OtsERC20TransferCounter); err != nil { + return err + } + if err := api.checkTransferIntegrity(tx, kv.OtsERC721TransferIndex, kv.OtsERC721TransferCounter); err != nil { + return err + } + return nil +} + +func (api *Otterscan2APIImpl) checkTransferIntegrity(tx kv.Tx, indexBucket, counterBucket string) error { + index, err := tx.Cursor(indexBucket) + if err != nil { + return err + } + defer index.Close() + + counter, err := tx.CursorDupSort(counterBucket) + if err != nil { + return err + } + defer counter.Close() + + k, v, err := index.First() + if err != nil { + return err + } + ck, cv, err := counter.First() + if err != nil { + return err + } + + var prevAddr []byte + addrCount := uint64(0) + expectedAddrCount := uint64(0) + accCount := uint64(0) + maxPrev := uint64(0) + + for k != nil { + addr := k[:length.Addr] + chunk := k[length.Addr:] + newCount := uint64(0) + counterChunk := []byte(nil) + + shouldBeSingleChunk := false + + // Address must match on index and counter + if !bytes.Equal(addr, ck) { + log.Warn("Integrity checker: counter bucket has different address", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + + changedAddress := !bytes.Equal(prevAddr, addr) + if changedAddress { + expectedAddrCount, err = counter.CountDuplicates() + if err != nil { + return err + } + shouldBeSingleChunk = expectedAddrCount == 1 + addrCount = 0 + accCount = 0 + maxPrev = 0 + } + addrCount++ + + // Chunk must match on index and chunk + if len(cv) != length.Counter+length.Chunk { + // Optimization is only allowed on 1st and unique counter of each address + if !changedAddress || !shouldBeSingleChunk { + log.Warn("Integrity checker: counter is optimized, but has multiple duplicates", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv), "c", expectedAddrCount) + return nil + } + if len(cv) != 1 { + log.Warn("Integrity checker: invalid optimized counter format", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + + newCount = uint64(cv[0]) + 1 + counterChunk = make([]byte, length.Chunk) + binary.BigEndian.PutUint64(counterChunk, ^uint64(0)) + } else { + newCount = binary.BigEndian.Uint64(cv[:length.Counter]) + counterChunk = cv[length.Counter:] + } + if !bytes.Equal(chunk, counterChunk) { + log.Warn("Integrity checker: chunks don't match", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + + // Chunk data must obey standard format + // Multiple of 8 bytes + if uint64(len(v))%8 != 0 { + log.Warn("Integrity checker: chunk bucket has multiple of 8 data", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + // Last id must be equal to chunk (unless it is 0xffff...) + if binary.BigEndian.Uint64(chunk) != ^uint64(0) && !bytes.Equal(chunk, v[len(v)-length.Chunk:]) { + log.Warn("Integrity checker: last value in chunk doesn't match", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + // Last chunk must be 0xffff... + if addrCount == expectedAddrCount && binary.BigEndian.Uint64(chunk) != ^uint64(0) { + log.Warn("Integrity checker: last chunk is not 0xffff...", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + // Mid-chunk can't be 0xffff.... + if addrCount != expectedAddrCount && binary.BigEndian.Uint64(chunk) == ^uint64(0) { + log.Warn("Integrity checker: mid chunk can't be 0xffff...", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + + // Prev count + card must be == newCount + card := uint64(len(v)) / 8 + if accCount+card != newCount { + log.Warn("Integrity checker: new count doesn't match", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv), "card", card, "prev", accCount) + return nil + } + accCount = newCount + + // Examine data inside chunk, must be sorted asc + for i := 0; i < len(v); i += 8 { + val := binary.BigEndian.Uint64(v[i : i+8]) + if val <= maxPrev { + log.Warn("Integrity checker: chunk data is not sorted asc", "index", indexBucket, "counter", counterBucket, "k", hexutility.Encode(k), "v", hexutility.Encode(v), "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + return nil + } + maxPrev = val + } + + // Next index + prevAddr = addr + k, v, err = index.Next() + if err != nil { + return err + } + + // Next counter + ck, cv, err = counter.NextDup() + if err != nil { + return err + } + if ck == nil { + ck, cv, err = counter.NextNoDup() + if err != nil { + return err + } + } + } + if ck != nil { + log.Warn("Integrity checker: index bucket EOF, counter bucket still has data", "index", indexBucket, "counter", counterBucket, "ck", hexutility.Encode(ck), "cv", hexutility.Encode(cv)) + } + log.Info("Integrity checker finished") + + return nil +} + +func (api *Otterscan2APIImpl) HoldingsIntegrityChecker(ctx context.Context) error { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return err + } + defer tx.Rollback() + + if err := api.checkHoldingsIntegrity(ctx, tx, kv.OtsERC20Holdings); err != nil { + return err + } + if err := api.checkHoldingsIntegrity(ctx, tx, kv.OtsERC721Holdings); err != nil { + return err + } + return nil +} + +func (api *Otterscan2APIImpl) checkHoldingsIntegrity(ctx context.Context, tx kv.Tx, indexBucket string) error { + index, err := tx.CursorDupSort(indexBucket) + if err != nil { + return err + } + defer index.Close() + + blockNum, err := stages.GetStageProgress(tx, stages.OtsERC20And721Holdings) + if err != nil { + return err + } + + baseTxId, err := api._blockReader.BaseTxIdForBlock(ctx, tx, blockNum) + if err != nil { + return err + } + block, err := api._blockReader.BlockByNumber(ctx, tx, blockNum) + if err != nil { + return err + } + txCount := block.Transactions().Len() + baseTxId += uint64(txCount + 2) + log.Info("xxx", "blockNum", blockNum, "txCount", txCount, "baseTxIdPlusOne", baseTxId) + + k, v, err := index.First() + if err != nil { + return err + } + + for k != nil { + ethTx := binary.BigEndian.Uint64(v[length.Addr:]) + if ethTx >= baseTxId { + return fmt.Errorf("integrity checker: stageBlockNum=%d bucket=%s holder=%s token=%s ethTx=%d maxEthTx=%d", blockNum, indexBucket, hexutility.Encode(k), hexutility.Encode(v[:length.Addr]), ethTx, baseTxId) + } + + k, v, err = index.NextDup() + if err != nil { + return err + } + if k == nil { + k, v, err = index.NextNoDup() + if err != nil { + return err + } + } + + select { + default: + case <-ctx.Done(): + return common.ErrStopped + } + } + + log.Info("Integrity checker finished") + return nil +} diff --git a/turbo/jsonrpc/otterscan2_erc1155.go b/turbo/jsonrpc/otterscan2_erc1155.go new file mode 100644 index 00000000000..dcb05bb9cc9 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc1155.go @@ -0,0 +1,103 @@ +package jsonrpc + +import ( + "bytes" + "context" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +type ERC1155Match struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` +} + +func (api *Otterscan2APIImpl) GetERC1155List(ctx context.Context, idx, count uint64) (*ContractListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + res, err := api.genericMatchingList(ctx, tx, kv.OtsERC1155, kv.OtsERC1155Counter, idx, count) + if err != nil { + return nil, err + } + + extraData, err := api.newERC1155ExtraData(ctx) + if err != nil { + return nil, err + } + + results, err := api.genericExtraData(ctx, tx, res, extraData) + if err != nil { + return nil, err + } + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, ToBlockSlice(res)) + if err != nil { + return nil, err + } + return &ContractListResult{ + BlocksSummary: blocksSummary, + Results: results, + }, nil +} + +func (api *Otterscan2APIImpl) newERC1155ExtraData(ctx context.Context) (ExtraDataExtractor, error) { + erc1155ABI, err := abi.JSON(bytes.NewReader(otscontracts.ERC20)) + if err != nil { + return nil, err + } + + name, err := erc1155ABI.Pack("name") + if err != nil { + return nil, err + } + symbol, err := erc1155ABI.Pack("symbol") + if err != nil { + return nil, err + } + + return func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) { + // name() + retName, err := decodeReturnData(ctx, &addr, name, "name", header, evm, chainConfig, ibs, &erc1155ABI) + if err != nil { + return nil, err + } + strName := "" + if retName != nil { + strName = retName.(string) + } + + // symbol() + retSymbol, err := decodeReturnData(ctx, &addr, symbol, "symbol", header, evm, chainConfig, ibs, &erc1155ABI) + if err != nil { + return nil, err + } + strSymbol := "" + if retSymbol != nil { + strSymbol = retSymbol.(string) + } + + return &ERC1155Match{ + res.Block, + res.Address, + strName, + strSymbol, + }, nil + }, nil +} + +func (api *Otterscan2APIImpl) GetERC1155Count(ctx context.Context) (uint64, error) { + return api.genericMatchingCounter(ctx, kv.OtsERC1155Counter) +} diff --git a/turbo/jsonrpc/otterscan2_erc1167.go b/turbo/jsonrpc/otterscan2_erc1167.go new file mode 100644 index 00000000000..407e57607e0 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc1167.go @@ -0,0 +1,104 @@ +package jsonrpc + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/rpc" + "github.com/ledgerwatch/erigon/turbo/rpchelper" +) + +type ERC1167Match struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` + Implementation *common.Address `json:"implementation"` +} + +func (api *Otterscan2APIImpl) GetERC1167List(ctx context.Context, idx, count uint64) (*ContractListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + res, err := api.genericMatchingList(ctx, tx, kv.OtsERC1167, kv.OtsERC1167Counter, idx, count) + if err != nil { + return nil, err + } + + extraData, err := api.newERC1167ExtraData(ctx) + if err != nil { + return nil, err + } + + results, err := api.genericExtraData(ctx, tx, res, extraData) + if err != nil { + return nil, err + } + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, ToBlockSlice(res)) + if err != nil { + return nil, err + } + return &ContractListResult{ + BlocksSummary: blocksSummary, + Results: results, + }, nil +} + +func (api *Otterscan2APIImpl) newERC1167ExtraData(ctx context.Context) (ExtraDataExtractor, error) { + return func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) { + acc, err := stateReader.ReadAccountData(addr) + if err != nil { + return nil, err + } + code, err := stateReader.ReadAccountCode(addr, acc.Incarnation, acc.CodeHash) + if err != nil { + return nil, err + } + + impl := common.BytesToAddress(code[10:30]) + + return &ERC1167Match{ + res.Block, + res.Address, + &impl, + }, nil + }, nil +} + +func (api *Otterscan2APIImpl) GetERC1167Count(ctx context.Context) (uint64, error) { + return api.genericMatchingCounter(ctx, kv.OtsERC1167Counter) +} + +func (api *Otterscan2APIImpl) GetERC1167Impl(ctx context.Context, addr common.Address) (common.Address, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return common.Address{}, err + } + defer tx.Rollback() + + chainConfig, err := api.chainConfig(tx) + if err != nil { + return common.Address{}, err + } + reader, err := rpchelper.CreateStateReader(ctx, tx, rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), 0, api.filters, api.stateCache, api.historyV3(tx), chainConfig.ChainName) + if err != nil { + return common.Address{}, err + } + acc, err := reader.ReadAccountData(addr) + if err != nil { + return common.Address{}, err + } + code, err := reader.ReadAccountCode(addr, acc.Incarnation, acc.CodeHash) + if err != nil { + return common.Address{}, err + } + + return common.BytesToAddress(code[10:30]), nil +} diff --git a/turbo/jsonrpc/otterscan2_erc20.go b/turbo/jsonrpc/otterscan2_erc20.go new file mode 100644 index 00000000000..23bf8f3400d --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc20.go @@ -0,0 +1,119 @@ +package jsonrpc + +import ( + "bytes" + "context" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +type ERC20Match struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals uint8 `json:"decimals"` +} + +func (api *Otterscan2APIImpl) GetERC20List(ctx context.Context, idx, count uint64) (*ContractListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + res, err := api.genericMatchingList(ctx, tx, kv.OtsERC20, kv.OtsERC20Counter, idx, count) + if err != nil { + return nil, err + } + + extraData, err := api.newERC20ExtraData(ctx) + if err != nil { + return nil, err + } + + results, err := api.genericExtraData(ctx, tx, res, extraData) + if err != nil { + return nil, err + } + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, ToBlockSlice(res)) + if err != nil { + return nil, err + } + return &ContractListResult{ + BlocksSummary: blocksSummary, + Results: results, + }, nil +} + +func (api *Otterscan2APIImpl) newERC20ExtraData(ctx context.Context) (ExtraDataExtractor, error) { + erc20ABI, err := abi.JSON(bytes.NewReader(otscontracts.ERC20)) + if err != nil { + return nil, err + } + + name, err := erc20ABI.Pack("name") + if err != nil { + return nil, err + } + symbol, err := erc20ABI.Pack("symbol") + if err != nil { + return nil, err + } + decimals, err := erc20ABI.Pack("decimals") + if err != nil { + return nil, err + } + + return func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) { + // name() + retName, err := decodeReturnData(ctx, &addr, name, "name", header, evm, chainConfig, ibs, &erc20ABI) + if err != nil { + return nil, err + } + strName := "" + if retName != nil { + strName = retName.(string) + } + + // symbol() + retSymbol, err := decodeReturnData(ctx, &addr, symbol, "symbol", header, evm, chainConfig, ibs, &erc20ABI) + if err != nil { + return nil, err + } + strSymbol := "" + if retSymbol != nil { + strSymbol = retSymbol.(string) + } + + // decimals() + retDecimals, err := decodeReturnData(ctx, &addr, decimals, "decimals", header, evm, chainConfig, ibs, &erc20ABI) + if err != nil { + return nil, err + } + nDecimals := uint8(0) + if retDecimals != nil { + nDecimals = retDecimals.(uint8) + } + + return &ERC20Match{ + res.Block, + res.Address, + strName, + strSymbol, + nDecimals, + }, nil + }, nil +} + +func (api *Otterscan2APIImpl) GetERC20Count(ctx context.Context) (uint64, error) { + return api.genericMatchingCounter(ctx, kv.OtsERC20Counter) +} diff --git a/turbo/jsonrpc/otterscan2_erc20_721_holdings.go b/turbo/jsonrpc/otterscan2_erc20_721_holdings.go new file mode 100644 index 00000000000..676b95f08b4 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc20_721_holdings.go @@ -0,0 +1,63 @@ +package jsonrpc + +import ( + "context" + "encoding/binary" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/length" + "github.com/ledgerwatch/erigon-lib/kv" +) + +type HoldingMatch struct { + Address common.Address `json:"address"` + Tx uint64 `json:"ethTx"` +} + +func (api *Otterscan2APIImpl) GetERC20Holdings(ctx context.Context, holder common.Address) ([]*HoldingMatch, error) { + return api.getHoldings(ctx, holder, kv.OtsERC20Holdings) +} + +func (api *Otterscan2APIImpl) GetERC721Holdings(ctx context.Context, holder common.Address) ([]*HoldingMatch, error) { + return api.getHoldings(ctx, holder, kv.OtsERC721Holdings) +} + +func (api *Otterscan2APIImpl) getHoldings(ctx context.Context, holder common.Address, holdingsBucket string) ([]*HoldingMatch, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, nil + } + defer tx.Rollback() + + holdings, err := tx.CursorDupSort(holdingsBucket) + if err != nil { + return nil, err + } + defer holdings.Close() + + k, v, err := holdings.SeekExact(holder.Bytes()) + if err != nil { + return nil, err + } + if k == nil { + return make([]*HoldingMatch, 0), nil + } + + ret := make([]*HoldingMatch, 0) + for k != nil { + token := common.BytesToAddress(v[:length.Addr]) + ethTx := binary.BigEndian.Uint64(v[length.Addr:]) + + ret = append(ret, &HoldingMatch{ + Address: token, + Tx: ethTx, + }) + + k, v, err = holdings.NextDup() + if err != nil { + return nil, err + } + } + + return ret, nil +} diff --git a/turbo/jsonrpc/otterscan2_erc20_721_transfers.go b/turbo/jsonrpc/otterscan2_erc20_721_transfers.go new file mode 100644 index 00000000000..67d3a38b40e --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc20_721_transfers.go @@ -0,0 +1,24 @@ +package jsonrpc + +import ( + "context" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/kv" +) + +func (api *Otterscan2APIImpl) GetERC20TransferList(ctx context.Context, addr common.Address, idx, count uint64) (*TransactionListResult, error) { + return api.genericTransferList(ctx, addr, idx, count, kv.OtsERC20TransferIndex, kv.OtsERC20TransferCounter) +} + +func (api *Otterscan2APIImpl) GetERC20TransferCount(ctx context.Context, addr common.Address) (uint64, error) { + return api.genericGetCount(ctx, addr, kv.OtsERC20TransferCounter) +} + +func (api *Otterscan2APIImpl) GetERC721TransferList(ctx context.Context, addr common.Address, idx, count uint64) (*TransactionListResult, error) { + return api.genericTransferList(ctx, addr, idx, count, kv.OtsERC721TransferIndex, kv.OtsERC721TransferCounter) +} + +func (api *Otterscan2APIImpl) GetERC721TransferCount(ctx context.Context, addr common.Address) (uint64, error) { + return api.genericGetCount(ctx, addr, kv.OtsERC721TransferCounter) +} diff --git a/turbo/jsonrpc/otterscan2_erc4626.go b/turbo/jsonrpc/otterscan2_erc4626.go new file mode 100644 index 00000000000..8029dfa32c9 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc4626.go @@ -0,0 +1,119 @@ +package jsonrpc + +import ( + "bytes" + "context" + "math/big" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +type ERC4626Match struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` + Decimals uint8 `json:"decimals"` + Asset common.Address `json:"asset"` + TotalAssets *big.Int `json:"totalAssets"` +} + +func (api *Otterscan2APIImpl) GetERC4626List(ctx context.Context, idx, count uint64) (*ContractListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + res, err := api.genericMatchingList(ctx, tx, kv.OtsERC4626, kv.OtsERC4626Counter, idx, count) + if err != nil { + return nil, err + } + + extraData, err := api.newERC4626ExtraData(ctx) + if err != nil { + return nil, err + } + + results, err := api.genericExtraData(ctx, tx, res, extraData) + if err != nil { + return nil, err + } + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, ToBlockSlice(res)) + if err != nil { + return nil, err + } + return &ContractListResult{ + BlocksSummary: blocksSummary, + Results: results, + }, nil +} + +func (api *Otterscan2APIImpl) newERC4626ExtraData(ctx context.Context) (ExtraDataExtractor, error) { + erc4626ABI, err := abi.JSON(bytes.NewReader(otscontracts.IERC4626)) + if err != nil { + return nil, err + } + + asset, err := erc4626ABI.Pack("asset") + if err != nil { + return nil, err + } + totalAssets, err := erc4626ABI.Pack("totalAssets") + if err != nil { + return nil, err + } + + return func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) { + erc20Extra, err := api.newERC20ExtraData(ctx) + if err != nil { + return nil, err + } + erc20Match, err := erc20Extra(tx, res, addr, evm, header, chainConfig, ibs, stateReader) + if err != nil { + return nil, err + } + + // asset() + retAsset, err := decodeReturnData(ctx, &addr, asset, "asset", header, evm, chainConfig, ibs, &erc4626ABI) + if err != nil { + return nil, err + } + addrAsset := common.Address{} + if retAsset != nil { + addrAsset = retAsset.(common.Address) + } + + // totalAssets() + retTotalAssets, err := decodeReturnData(ctx, &addr, totalAssets, "totalAssets", header, evm, chainConfig, ibs, &erc4626ABI) + if err != nil { + return nil, err + } + var nTotalAssets *big.Int + if retTotalAssets != nil { + nTotalAssets = retTotalAssets.(*big.Int) + } + + return &ERC4626Match{ + Block: erc20Match.(*ERC20Match).Block, + Address: erc20Match.(*ERC20Match).Address, + Name: erc20Match.(*ERC20Match).Name, + Symbol: erc20Match.(*ERC20Match).Symbol, + Decimals: erc20Match.(*ERC20Match).Decimals, + Asset: addrAsset, + TotalAssets: nTotalAssets, + }, nil + }, nil +} + +func (api *Otterscan2APIImpl) GetERC4626Count(ctx context.Context) (uint64, error) { + return api.genericMatchingCounter(ctx, kv.OtsERC4626Counter) +} diff --git a/turbo/jsonrpc/otterscan2_erc721.go b/turbo/jsonrpc/otterscan2_erc721.go new file mode 100644 index 00000000000..4f6966ba347 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_erc721.go @@ -0,0 +1,103 @@ +package jsonrpc + +import ( + "bytes" + "context" + + "github.com/ledgerwatch/erigon-lib/chain" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/accounts/abi" + "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/eth/stagedsync/otscontracts" +) + +type ERC721Match struct { + Block *hexutil.Uint64 `json:"blockNumber"` + Address *common.Address `json:"address"` + Name string `json:"name"` + Symbol string `json:"symbol"` +} + +func (api *Otterscan2APIImpl) GetERC721List(ctx context.Context, idx, count uint64) (*ContractListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + res, err := api.genericMatchingList(ctx, tx, kv.OtsERC721, kv.OtsERC721Counter, idx, count) + if err != nil { + return nil, err + } + + extraData, err := api.newERC721ExtraData(ctx) + if err != nil { + return nil, err + } + + results, err := api.genericExtraData(ctx, tx, res, extraData) + if err != nil { + return nil, err + } + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, ToBlockSlice(res)) + if err != nil { + return nil, err + } + return &ContractListResult{ + BlocksSummary: blocksSummary, + Results: results, + }, nil +} + +func (api *Otterscan2APIImpl) newERC721ExtraData(ctx context.Context) (ExtraDataExtractor, error) { + erc721ABI, err := abi.JSON(bytes.NewReader(otscontracts.ERC20)) + if err != nil { + return nil, err + } + + name, err := erc721ABI.Pack("name") + if err != nil { + return nil, err + } + symbol, err := erc721ABI.Pack("symbol") + if err != nil { + return nil, err + } + + return func(tx kv.Tx, res *AddrMatch, addr common.Address, evm *vm.EVM, header *types.Header, chainConfig *chain.Config, ibs *state.IntraBlockState, stateReader state.StateReader) (interface{}, error) { + // name() + retName, err := decodeReturnData(ctx, &addr, name, "name", header, evm, chainConfig, ibs, &erc721ABI) + if err != nil { + return nil, err + } + strName := "" + if retName != nil { + strName = retName.(string) + } + + // symbol() + retSymbol, err := decodeReturnData(ctx, &addr, symbol, "symbol", header, evm, chainConfig, ibs, &erc721ABI) + if err != nil { + return nil, err + } + strSymbol := "" + if retSymbol != nil { + strSymbol = retSymbol.(string) + } + + return &ERC721Match{ + res.Block, + res.Address, + strName, + strSymbol, + }, nil + }, nil +} + +func (api *Otterscan2APIImpl) GetERC721Count(ctx context.Context) (uint64, error) { + return api.genericMatchingCounter(ctx, kv.OtsERC721Counter) +} diff --git a/turbo/jsonrpc/otterscan2_withdrawals.go b/turbo/jsonrpc/otterscan2_withdrawals.go new file mode 100644 index 00000000000..4af93b71a06 --- /dev/null +++ b/turbo/jsonrpc/otterscan2_withdrawals.go @@ -0,0 +1,126 @@ +package jsonrpc + +import ( + "context" + "encoding/binary" + + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/hexutil" + "github.com/ledgerwatch/erigon-lib/common/hexutility" + "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon/core/types" + "github.com/ledgerwatch/erigon/turbo/services" +) + +type WithdrawalsListResult struct { + BlocksSummary map[hexutil.Uint64]*BlockSummary `json:"blocksSummary"` + Results []*WithdrawalMatch `json:"results"` +} + +type WithdrawalMatch struct { + Index hexutil.Uint64 `json:"index"` + BlockNum hexutil.Uint64 `json:"blockNumber"` + Validator hexutil.Uint64 `json:"validatorIndex"` + Amount hexutil.Uint64 `json:"amount"` +} + +type withdrawalsSearchResultMaterializer struct { + blockReader services.FullBlockReader + idx2Block kv.Cursor +} + +func NewWithdrawalsSearchResultMaterializer(tx kv.Tx, blockReader services.FullBlockReader) (*withdrawalsSearchResultMaterializer, error) { + idx2Block, err := tx.Cursor(kv.OtsWithdrawalIdx2Block) + if err != nil { + return nil, err + } + + return &withdrawalsSearchResultMaterializer{blockReader, idx2Block}, nil +} + +func (w *withdrawalsSearchResultMaterializer) Convert(ctx context.Context, tx kv.Tx, idx uint64) (*WithdrawalMatch, error) { + idx2Block, err := tx.Cursor(kv.OtsWithdrawalIdx2Block) + if err != nil { + return nil, err + } + defer idx2Block.Close() + + k, v, err := idx2Block.Seek(hexutility.EncodeTs(idx)) + if err != nil { + return nil, err + } + if k == nil { + return nil, nil + } + + blockNum := binary.BigEndian.Uint64(v) + hash, err := w.blockReader.CanonicalHash(ctx, tx, blockNum) + if err != nil { + return nil, err + } + body, _, err := w.blockReader.Body(ctx, tx, hash, blockNum) + if err != nil { + return nil, err + } + + var match *types.Withdrawal + for _, w := range body.Withdrawals { + if w.Index == idx { + match = w + break + } + } + if match == nil { + // TODO: error + return nil, nil + } + + result := &WithdrawalMatch{ + Index: hexutil.Uint64(idx), + BlockNum: hexutil.Uint64(blockNum), + Validator: hexutil.Uint64(match.Validator), + Amount: hexutil.Uint64(match.Amount), + } + return result, nil +} + +func (w *withdrawalsSearchResultMaterializer) Dispose() { + w.idx2Block.Close() +} + +func (api *Otterscan2APIImpl) GetWithdrawalsList(ctx context.Context, addr common.Address, idx, count uint64) (*WithdrawalsListResult, error) { + tx, err := api.db.BeginRo(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback() + + srm, err := NewWithdrawalsSearchResultMaterializer(tx, api._blockReader) + if err != nil { + return nil, err + } + defer srm.Dispose() + + ret, err := genericResultList(ctx, tx, addr, idx, count, kv.OtsWithdrawalsIndex, kv.OtsWithdrawalsCounter, (SearchResultMaterializer[WithdrawalMatch])(srm)) + if err != nil { + return nil, err + } + + blocks := make([]hexutil.Uint64, 0, len(ret)) + for _, r := range ret { + blocks = append(blocks, hexutil.Uint64(r.BlockNum)) + } + + blocksSummary, err := api.newBlocksSummaryFromResults(ctx, tx, blocks) + if err != nil { + return nil, err + } + return &WithdrawalsListResult{ + BlocksSummary: blocksSummary, + Results: ret, + }, nil +} + +func (api *Otterscan2APIImpl) GetWithdrawalsCount(ctx context.Context, addr common.Address) (uint64, error) { + return api.genericGetCount(ctx, addr, kv.OtsWithdrawalsCounter) +} diff --git a/turbo/jsonrpc/otterscan_api.go b/turbo/jsonrpc/otterscan_api.go index 9b14b8588f2..9014792e8bd 100644 --- a/turbo/jsonrpc/otterscan_api.go +++ b/turbo/jsonrpc/otterscan_api.go @@ -10,7 +10,6 @@ import ( "golang.org/x/sync/errgroup" "github.com/holiman/uint256" - "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon-lib/chain" "github.com/ledgerwatch/erigon-lib/common" @@ -28,6 +27,7 @@ import ( "github.com/ledgerwatch/erigon/turbo/adapter/ethapi" "github.com/ledgerwatch/erigon/turbo/rpchelper" "github.com/ledgerwatch/erigon/turbo/transactions" + "github.com/ledgerwatch/log/v3" ) // API_LEVEL Must be incremented every time new additions are made diff --git a/turbo/services/interfaces.go b/turbo/services/interfaces.go index 0857520ac88..de9772d873f 100644 --- a/turbo/services/interfaces.go +++ b/turbo/services/interfaces.go @@ -58,7 +58,11 @@ type TxnReader interface { TxnLookup(ctx context.Context, tx kv.Getter, txnHash common.Hash) (uint64, bool, error) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, i int) (txn types.Transaction, err error) RawTransactions(ctx context.Context, tx kv.Getter, fromBlock, toBlock uint64) (txs [][]byte, err error) + TxnByTxId(ctx context.Context, tx kv.Getter, txId uint64) (txn types.Transaction, err error) + TxIdByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, i int) (txid uint64, err error) + BaseTxIdForBlock(ctx context.Context, tx kv.Getter, blockNum uint64) (txid uint64, err error) } + type HeaderAndCanonicalReader interface { HeaderReader CanonicalReader diff --git a/turbo/snapshotsync/freezeblocks/block_reader.go b/turbo/snapshotsync/freezeblocks/block_reader.go index fc6834937fa..b4007e0e7f7 100644 --- a/turbo/snapshotsync/freezeblocks/block_reader.go +++ b/turbo/snapshotsync/freezeblocks/block_reader.go @@ -145,6 +145,18 @@ func (r *RemoteBlockReader) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, b return b.Transactions[i], nil } +func (r *RemoteBlockReader) TxnByTxId(ctx context.Context, tx kv.Getter, txId uint64) (txn types.Transaction, err error) { + panic("not implemented") +} + +func (r *RemoteBlockReader) TxIdByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, i int) (txid uint64, err error) { + panic("not implemented") +} + +func (r *RemoteBlockReader) BaseTxIdForBlock(ctx context.Context, tx kv.Getter, blockNum uint64) (txid uint64, err error) { + panic("not implemented") +} + func (r *RemoteBlockReader) HasSenders(ctx context.Context, _ kv.Getter, hash common.Hash, blockHeight uint64) (bool, error) { panic("HasSenders is low-level method, don't use it in RPCDaemon") } @@ -812,6 +824,93 @@ func (r *BlockReader) TxnByIdxInBlock(ctx context.Context, tx kv.Getter, blockNu return r.txnByID(b.BaseTxId+1+uint64(txIdxInBlock), txnSeg, nil) } +func (r *BlockReader) TxnByTxId(ctx context.Context, tx kv.Getter, txId uint64) (txn types.Transaction, err error) { + blocksAvailable := r.sn.BlocksAvailable() + view := r.sn.View() + defer view.Close() + + // Determine the max baseTxId from highest snapshot body in order to + // decide if the desired txId is on DB or snapshots + max := uint64(0) + if blocksAvailable > 0 { + seg, ok := view.BodiesSegment(blocksAvailable) + if !ok { + return nil, nil + } + b, _, err := r.bodyForStorageFromSnapshot(blocksAvailable, seg, nil) + if err != nil { + return nil, err + } + max = b.BaseTxId + 2 + uint64(b.TxAmount) + } + + // Tx is in the DB + if txId > max { + txs, err := rawdb.CanonicalTransactions(tx, txId, 1) + if err != nil { + return nil, err + } + return txs[0], nil + } + + // Tx in in snapshots + for _, seg := range view.Txs() { + if txId >= seg.IdxTxnHash.BaseDataID() && txId < seg.IdxTxnHash.BaseDataID()+seg.IdxTxnHash.KeyCount() { + txn, err = r.txnByID(txId, seg, nil) + if err != nil { + return nil, err + } + return txn, nil + } + } + return nil, nil +} + +func (r *BlockReader) TxIdByIdxInBlock(ctx context.Context, tx kv.Getter, blockNum uint64, i int) (txid uint64, err error) { + baseTxid, err := r.BaseTxIdForBlock(ctx, tx, blockNum) + if err != nil { + return 0, err + } + return baseTxid + 1 + uint64(i), nil +} + +func (r *BlockReader) BaseTxIdForBlock(ctx context.Context, tx kv.Getter, blockNum uint64) (txid uint64, err error) { + blocksAvailable := r.sn.BlocksAvailable() + if blocksAvailable == 0 || blockNum > blocksAvailable { + canonicalHash, err := rawdb.ReadCanonicalHash(tx, blockNum) + if err != nil { + return 0, err + } + + var k [length.BlockNum + length.Hash]byte + binary.BigEndian.PutUint64(k[:], blockNum) + copy(k[length.BlockNum:], canonicalHash[:]) + + b, err := rawdb.ReadBodyForStorageByKey(tx, k[:]) + if err != nil { + return 0, err + } + if b == nil { + return 0, nil + } + + return b.BaseTxId, nil + } + + view := r.sn.View() + defer view.Close() + seg, ok := view.BodiesSegment(blockNum) + if !ok { + return 0, nil + } + + b, _, err := r.bodyForStorageFromSnapshot(blockNum, seg, nil) + if err != nil { + return 0, err + } + return b.BaseTxId, nil +} + // TxnLookup - find blockNumber and txnID by txnHash func (r *BlockReader) TxnLookup(ctx context.Context, tx kv.Getter, txnHash common.Hash) (uint64, bool, error) { n, err := rawdb.ReadTxLookupEntry(tx, txnHash) diff --git a/turbo/stages/mock/mock_sentry.go b/turbo/stages/mock/mock_sentry.go index 6feee7c5994..12d83dc3009 100644 --- a/turbo/stages/mock/mock_sentry.go +++ b/turbo/stages/mock/mock_sentry.go @@ -467,6 +467,8 @@ func MockWithEverything(tb testing.TB, gspec *types.Genesis, key *ecdsa.PrivateK stagedsync.StageCallTracesCfg(mock.DB, prune, 0, dirs.Tmp), stagedsync.StageTxLookupCfg(mock.DB, prune, dirs.Tmp, mock.ChainConfig.Bor, mock.BlockReader), stagedsync.StageFinishCfg(mock.DB, dirs.Tmp, forkValidator), + stagedsync.StageDbAwareCfg(mock.DB, dirs.Tmp, mock.ChainConfig, mock.BlockReader, mock.Engine), + false, !withPosDownloader), stagedsync.DefaultUnwindOrder, stagedsync.DefaultPruneOrder, diff --git a/turbo/stages/stageloop.go b/turbo/stages/stageloop.go index 35aab454bcd..2e4fc7fcff7 100644 --- a/turbo/stages/stageloop.go +++ b/turbo/stages/stageloop.go @@ -110,6 +110,10 @@ func StageLoopIteration(ctx context.Context, db kv.RwDB, txc wrap.TxContainer, s }() // avoid crash because Erigon's core does many things externalTx := txc.Tx != nil + otsProgressBefore, err := stagesOtsIndexers(db, txc.Tx) + if err != nil { + return err + } finishProgressBefore, borProgressBefore, headersProgressBefore, err := stagesHeadersAndFinish(db, txc.Tx) if err != nil { return err @@ -117,7 +121,7 @@ func StageLoopIteration(ctx context.Context, db kv.RwDB, txc wrap.TxContainer, s // Sync from scratch must be able Commit partial progress // In all other cases - process blocks batch in 1 RwTx // 2 corner-cases: when sync with --snapshots=false and when executed only blocks from snapshots (in this case all stages progress is equal and > 0, but node is not synced) - isSynced := finishProgressBefore > 0 && finishProgressBefore > blockReader.FrozenBlocks() && finishProgressBefore == headersProgressBefore + isSynced := finishProgressBefore > 0 && finishProgressBefore > blockReader.FrozenBlocks() && finishProgressBefore == headersProgressBefore && otsProgressBefore == headersProgressBefore if blockReader.BorSnapshots() != nil { isSynced = isSynced && borProgressBefore > blockReader.FrozenBorBlocks() } @@ -198,6 +202,26 @@ func stageLoopStepPrune(ctx context.Context, db kv.RwDB, tx kv.RwTx, sync *stage return db.Update(ctx, func(tx kv.RwTx) error { return sync.RunPrune(db, tx, initialCycle) }) } +var lastOtsStage = stages.OtsERC20And721Holdings + +func stagesOtsIndexers(db kv.RoDB, tx kv.Tx) (lastIdx uint64, err error) { + if tx != nil { + if lastIdx, err = stages.GetStageProgress(tx, lastOtsStage); err != nil { + return lastIdx, err + } + return lastIdx, nil + } + if err := db.View(context.Background(), func(tx kv.Tx) error { + if lastIdx, err = stages.GetStageProgress(tx, lastOtsStage); err != nil { + return err + } + return nil + }); err != nil { + return lastIdx, err + } + return lastIdx, nil +} + func stagesHeadersAndFinish(db kv.RoDB, tx kv.Tx) (head, bor, fin uint64, err error) { if tx != nil { if fin, err = stages.GetStageProgress(tx, stages.Finish); err != nil { @@ -541,6 +565,8 @@ func NewDefaultStages(ctx context.Context, stagedsync.StageCallTracesCfg(db, cfg.Prune, 0, dirs.Tmp), stagedsync.StageTxLookupCfg(db, cfg.Prune, dirs.Tmp, controlServer.ChainConfig.Bor, blockReader), stagedsync.StageFinishCfg(db, dirs.Tmp, forkValidator), + stagedsync.StageDbAwareCfg(db, dirs.Tmp, controlServer.ChainConfig, blockReader, controlServer.Engine), + cfg.Ots2, runInTestMode) } @@ -615,6 +641,8 @@ func NewPipelineStages(ctx context.Context, stagedsync.StageCallTracesCfg(db, cfg.Prune, 0, dirs.Tmp), stagedsync.StageTxLookupCfg(db, cfg.Prune, dirs.Tmp, controlServer.ChainConfig.Bor, blockReader), stagedsync.StageFinishCfg(db, dirs.Tmp, forkValidator), + stagedsync.StageDbAwareCfg(db, dirs.Tmp, controlServer.ChainConfig, blockReader, controlServer.Engine), + cfg.Ots2, runInTestMode) }