diff --git a/.editorconfig b/.editorconfig index 472016080b2..b8d038cba7f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -9,7 +9,7 @@ indent_style = space insert_final_newline = true trim_trailing_whitespace = true -file_header_template = SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited\nSPDX-License-Identifier: LGPL-3.0-only +file_header_template = SPDX-FileCopyrightText: 2024 Demerzel Solutions Limited\nSPDX-License-Identifier: LGPL-3.0-only [*.cs] indent_size = 4 @@ -34,31 +34,31 @@ dotnet_naming_rule.non_field_members_should_be_pascal_case.style = pascal_case dotnet_naming_symbols.interface.applicable_kinds = interface dotnet_naming_symbols.interface.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected -dotnet_naming_symbols.interface.required_modifiers = +dotnet_naming_symbols.interface.required_modifiers = dotnet_naming_symbols.types.applicable_kinds = class, struct, interface, enum dotnet_naming_symbols.types.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected -dotnet_naming_symbols.types.required_modifiers = +dotnet_naming_symbols.types.required_modifiers = dotnet_naming_symbols.non_field_members.applicable_kinds = property, event, method dotnet_naming_symbols.non_field_members.applicable_accessibilities = public, internal, private, protected, protected_internal, private_protected -dotnet_naming_symbols.non_field_members.required_modifiers = +dotnet_naming_symbols.non_field_members.required_modifiers = # Naming styles dotnet_naming_style.begins_with_i.required_prefix = I -dotnet_naming_style.begins_with_i.required_suffix = -dotnet_naming_style.begins_with_i.word_separator = +dotnet_naming_style.begins_with_i.required_suffix = +dotnet_naming_style.begins_with_i.word_separator = dotnet_naming_style.begins_with_i.capitalization = pascal_case -dotnet_naming_style.pascal_case.required_prefix = -dotnet_naming_style.pascal_case.required_suffix = -dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = dotnet_naming_style.pascal_case.capitalization = pascal_case -dotnet_naming_style.pascal_case.required_prefix = -dotnet_naming_style.pascal_case.required_suffix = -dotnet_naming_style.pascal_case.word_separator = +dotnet_naming_style.pascal_case.required_prefix = +dotnet_naming_style.pascal_case.required_suffix = +dotnet_naming_style.pascal_case.word_separator = dotnet_naming_style.pascal_case.capitalization = pascal_case dotnet_style_operator_placement_when_wrapping = beginning_of_line dotnet_style_coalesce_expression = true:suggestion @@ -88,4 +88,4 @@ csharp_style_expression_bodied_lambdas = true:silent csharp_style_expression_bodied_local_functions = false:silent csharp_style_var_for_built_in_types = true:silent csharp_style_var_when_type_is_apparent = true:suggestion -csharp_style_var_elsewhere = false:suggestion \ No newline at end of file +csharp_style_var_elsewhere = false:suggestion diff --git a/.github/workflows/build-nethermind-packages.yml b/.github/workflows/build-nethermind-packages.yml index 5dd1d48ab42..2595c718407 100644 --- a/.github/workflows/build-nethermind-packages.yml +++ b/.github/workflows/build-nethermind-packages.yml @@ -1,65 +1,49 @@ name: Build Nethermind packages -on: +on: workflow_dispatch: jobs: build: name: Build Nethermind packages runs-on: ubuntu-latest - env: + env: PACKAGE_DIR: pkg PACKAGE_RETENTION: 7 PUB_DIR: pub steps: - - name: Check out Nethermind repository + - name: Check out repository uses: actions/checkout@v4 - with: - path: nethermind - - name: Check out Nethermind Launcher repository - uses: actions/checkout@v4 - with: - repository: NethermindEth/nethermind.launcher - path: launcher - name: Set up .NET uses: actions/setup-dotnet@v4 - with: - global-json-file: nethermind/global.json - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 14 - - name: Install npm packages - run: npm i -g pkg @vercel/ncc - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - name: Build Nethermind.Runner id: build-runner run: | - cd nethermind build_timestamp=$(date '+%s') echo "build-timestamp=$build_timestamp" >> $GITHUB_OUTPUT echo "commit-hash=${GITHUB_SHA:0:8}" >> $GITHUB_OUTPUT scripts/deployment/build-runner.sh $GITHUB_SHA $build_timestamp - name: Build Nethermind.Cli - run: nethermind/scripts/deployment/build-cli.sh $GITHUB_SHA ${{ steps.build-runner.outputs.build-timestamp }} - - name: Build Nethermind launcher - run: nethermind/scripts/deployment/build-launcher.sh - - name: Build Nethermind launcher for Linux arm64 + run: scripts/deployment/build-cli.sh $GITHUB_SHA ${{ steps.build-runner.outputs.build-timestamp }} + - name: Bundle Nethermind launcher run: | - cd nethermind - docker buildx build --platform=linux/arm64 -t tmp-launcher -f Dockerfile.launcher . --load - docker run --platform=linux/arm64 -v $PWD:/opt/mount --rm tmp-launcher bash -c "cp /nethermind/nethermind-launcher /opt/mount/" - mv nethermind-launcher $GITHUB_WORKSPACE/$PUB_DIR/linux-arm64/nethermind-launcher + json=$(curl -s https://github.com/gitapi/repos/nethermindeth/nethermind.launcher/releases/latest) + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("linux-x64"))') + curl -sSL $url -o $PUB_DIR/linux-x64/nethermind-launcher + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("linux-arm64"))') + curl -sSL $url -o $PUB_DIR/linux-arm64/nethermind-launcher + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("windows-x64"))') + curl -sSL $url -o $PUB_DIR/win-x64/nethermind-launcher.exe + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("macos-x64"))') + curl -sSL $url -o $PUB_DIR/osx-x64/nethermind-launcher + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("macos-arm64"))') + curl -sSL $url -o $PUB_DIR/osx-arm64/nethermind-launcher - name: Archive packages env: PACKAGE_PREFIX: nethermind-preview-${{ steps.build-runner.outputs.commit-hash }} run: | echo "PACKAGE_PREFIX=$PACKAGE_PREFIX" >> $GITHUB_ENV - nethermind/scripts/deployment/archive-packages.sh + scripts/deployment/archive-packages.sh - name: Upload Nethermind Linux x64 package uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/build-solutions.yml b/.github/workflows/build-solutions.yml index 930bf01e379..f339c0f2920 100644 --- a/.github/workflows/build-solutions.yml +++ b/.github/workflows/build-solutions.yml @@ -6,38 +6,20 @@ on: push: branches: [master] -defaults: - run: - working-directory: src/Nethermind - -env: - BUILD_CONFIG: release - DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1 - TERM: xterm - jobs: build: name: Build runs-on: ubuntu-latest - permissions: - contents: read + strategy: + matrix: + config: [release, debug] + solution: [Nethermind, EthereumTests, Benchmarks] steps: - name: Check out repository uses: actions/checkout@v4 with: - submodules: true - - name: Install Linux packages - run: sudo apt-get update && sudo apt-get install libsnappy-dev + submodules: ${{ matrix.solution == 'EthereumTests' }} - name: Set up .NET uses: actions/setup-dotnet@v4 - - name: Install dependencies - run: | - dotnet restore Nethermind.sln - dotnet restore EthereumTests.sln - dotnet restore Benchmarks.sln - - name: Build Nethermind.sln - run: dotnet build Nethermind.sln -c ${{ env.BUILD_CONFIG }} --no-restore - - name: Build EthereumTests.sln - run: dotnet build EthereumTests.sln -c ${{ env.BUILD_CONFIG }} --no-restore - - name: Build Benchmarks.sln - run: dotnet build Benchmarks.sln -c ${{ env.BUILD_CONFIG }} --no-restore + - name: Build ${{ matrix.solution }}.sln + run: dotnet build src/Nethermind/${{ matrix.solution }}.sln -c ${{ matrix.config }} diff --git a/.github/workflows/posdao-tests.yml b/.github/workflows/posdao-tests.yml index 0433b9b0a32..55cd250decd 100644 --- a/.github/workflows/posdao-tests.yml +++ b/.github/workflows/posdao-tests.yml @@ -90,5 +90,5 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: posdao-logs + name: posdao-logs-${{ matrix.branch }} path: posdao-logs/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 01c810fa819..b344bb08321 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,6 @@ name: Release -on: +on: workflow_dispatch: inputs: tag: @@ -16,7 +16,7 @@ env: PACKAGE_DIR: pkg PACKAGE_RETENTION: 7 PUB_DIR: pub - SCRIPTS_PATH: ${{ github.workspace }}/nethermind/scripts/deployment + SCRIPTS_PATH: ${{ github.workspace }}/scripts/deployment jobs: build: @@ -27,30 +27,10 @@ jobs: package-prefix: ${{ steps.archive.outputs.package-prefix }} prerelease: ${{ steps.build-runner.outputs.prerelease }} steps: - - name: Check out Nethermind repository - uses: actions/checkout@v4 - with: - path: nethermind - - name: Check out Nethermind Launcher repository + - name: Check out repository uses: actions/checkout@v4 - with: - repository: NethermindEth/nethermind.launcher - path: launcher - name: Set up .NET uses: actions/setup-dotnet@v4 - with: - global-json-file: nethermind/global.json - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 14 - - name: Install npm packages - run: npm i -g pkg @vercel/ncc - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v3 - name: Build Nethermind.Runner id: build-runner run: | @@ -61,14 +41,19 @@ jobs: $SCRIPTS_PATH/build-runner.sh $GITHUB_SHA $build_timestamp - name: Build Nethermind.Cli run: $SCRIPTS_PATH/build-cli.sh $GITHUB_SHA ${{ steps.build-runner.outputs.build-timestamp }} - - name: Build Nethermind launcher - run: $SCRIPTS_PATH/build-launcher.sh - - name: Build Nethermind launcher for Linux arm64 - working-directory: nethermind + - name: Bundle Nethermind launcher run: | - docker buildx build --platform=linux/arm64 -t tmp-launcher -f Dockerfile.launcher . --load - docker run --platform=linux/arm64 -v $PWD:/opt/mount --rm tmp-launcher bash -c "cp /nethermind/nethermind-launcher /opt/mount/" - mv nethermind-launcher $GITHUB_WORKSPACE/$PUB_DIR/linux-arm64/nethermind-launcher + json=$(curl -s https://github.com/gitapi/repos/nethermindeth/nethermind.launcher/releases/latest) + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("linux-x64"))') + curl -sSL $url -o $PUB_DIR/linux-x64/nethermind-launcher + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("linux-arm64"))') + curl -sSL $url -o $PUB_DIR/linux-arm64/nethermind-launcher + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("windows-x64"))') + curl -sSL $url -o $PUB_DIR/win-x64/nethermind-launcher.exe + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("macos-x64"))') + curl -sSL $url -o $PUB_DIR/osx-x64/nethermind-launcher + url=$(echo "$json" | jq -r '.assets[].browser_download_url | select(contains("macos-arm64"))') + curl -sSL $url -o $PUB_DIR/osx-arm64/nethermind-launcher - name: Archive packages id: archive env: @@ -112,7 +97,7 @@ jobs: name: ${{ steps.archive.outputs.package-prefix }}-ref-assemblies-package path: ${{ github.workspace }}/${{ env.PACKAGE_DIR }}/*ref-assemblies* retention-days: ${{ env.PACKAGE_RETENTION }} - + approval: name: Approve runs-on: ubuntu-latest @@ -131,8 +116,6 @@ jobs: steps: - name: Check out Nethermind repository uses: actions/checkout@v4 - with: - path: nethermind - name: Download artifacts uses: actions/download-artifact@v4 with: @@ -156,8 +139,6 @@ jobs: steps: - name: Check out Nethermind repository uses: actions/checkout@v4 - with: - path: nethermind - name: Download artifacts uses: actions/download-artifact@v4 with: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 026af98679f..26ef4fd581a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,7 @@ Branch names must follow `snake_case` pattern. Follow the pattern `Demerzel Solutions Limited Nethermind $(Commit.Substring(0, 8)) - 1.24.0 + 1.26.0 unstable diff --git a/src/Nethermind/Directory.Packages.props b/src/Nethermind/Directory.Packages.props index c5ea08ce0dd..2f5d58aa8b5 100644 --- a/src/Nethermind/Directory.Packages.props +++ b/src/Nethermind/Directory.Packages.props @@ -4,12 +4,12 @@ - - - - - - + + + + + + @@ -23,7 +23,7 @@ - + @@ -48,14 +48,14 @@ - + - + @@ -64,15 +64,15 @@ - - + + - + @@ -85,9 +85,9 @@ - - + + - \ No newline at end of file + diff --git a/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationSubscribeTests.cs b/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationSubscribeTests.cs index 409ed0d0beb..799ee6444db 100644 --- a/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationSubscribeTests.cs +++ b/src/Nethermind/Nethermind.AccountAbstraction.Test/UserOperationSubscribeTests.cs @@ -62,7 +62,7 @@ public void Setup() _blockTree = Substitute.For(); _txPool = Substitute.For(); _receiptStorage = Substitute.For(); - _receiptCanonicalityMonitor = new ReceiptCanonicalityMonitor(_blockTree, _receiptStorage, _logManager); + _receiptCanonicalityMonitor = new ReceiptCanonicalityMonitor(_receiptStorage, _logManager); _specProvider = Substitute.For(); _userOperationPools[_testPoolAddress] = Substitute.For(); _filterStore = new FilterStore(); diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 12fe585bc59..8a5bfa981d5 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -15,7 +15,7 @@ namespace Nethermind.Api { public interface IApiWithStores : IBasicApi { - ITxStorage? BlobTxStorage { get; set; } + IBlobTxStorage? BlobTxStorage { get; set; } IBlockTree? BlockTree { get; set; } IBloomStorage? BloomStorage { get; set; } IChainLevelInfoRepository? ChainLevelInfoRepository { get; set; } diff --git a/src/Nethermind/Nethermind.Api/IInitConfig.cs b/src/Nethermind/Nethermind.Api/IInitConfig.cs index 6512ca3a823..ddc999b8b99 100644 --- a/src/Nethermind/Nethermind.Api/IInitConfig.cs +++ b/src/Nethermind/Nethermind.Api/IInitConfig.cs @@ -79,6 +79,9 @@ public interface IInitConfig : IConfig [ConfigItem(Description = "[TECHNICAL] Disable setting malloc options. Set to true if using different memory allocator or manually setting malloc opts.", DefaultValue = "false", HiddenFromDocs = true)] bool DisableMallocOpts { get; set; } + + [ConfigItem(Description = "[TECHNICAL] Exit when block number is reached. Useful for scripting and testing.", DefaultValue = "null", HiddenFromDocs = true)] + long? ExitOnBlockNumber { get; set; } } public enum DiagnosticMode diff --git a/src/Nethermind/Nethermind.Api/InitConfig.cs b/src/Nethermind/Nethermind.Api/InitConfig.cs index 676767780f7..af7a04c6971 100644 --- a/src/Nethermind/Nethermind.Api/InitConfig.cs +++ b/src/Nethermind/Nethermind.Api/InitConfig.cs @@ -33,6 +33,7 @@ public class InitConfig : IInitConfig public long? MemoryHint { get; set; } public bool DisableGcOnNewPayload { get; set; } = true; public bool DisableMallocOpts { get; set; } = false; + public long? ExitOnBlockNumber { get; set; } = null; [Obsolete("Use DiagnosticMode with MemDb instead")] public bool UseMemDb diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index 3574e6bbb92..373fa72e722 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -100,7 +100,7 @@ public IBlockchainBridge CreateBlockchainBridge() } public IAbiEncoder AbiEncoder { get; } = Nethermind.Abi.AbiEncoder.Instance; - public ITxStorage? BlobTxStorage { get; set; } + public IBlobTxStorage? BlobTxStorage { get; set; } public IBlockchainProcessor? BlockchainProcessor { get; set; } public CompositeBlockPreprocessorStep BlockPreprocessor { get; } = new(); public IBlockProcessingQueue? BlockProcessingQueue { get; set; } diff --git a/src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs index abf91c29860..17da8cffc11 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/BlockTreeTests.cs @@ -1827,8 +1827,9 @@ public void Find_handles_invalid_blocks(Func(); + IProcessExitSource processExitSource = Substitute.For(); + new ExitOnBlockNumberHandler(blockTree, processExitSource, 100, LimboLogs.Instance); + + blockTree.BlockAddedToMain += Raise.EventWith( + new BlockReplacementEventArgs(Build.A.Block.WithNumber(blockNumber).TestObject)); + + if (exitReceived) + { + processExitSource.Received().Exit(0); + } + else + { + processExitSource.DidNotReceive().Exit(0); + } + } +} diff --git a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/CopyTreeVisitorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/CopyTreeVisitorTests.cs index 7150f0fbabb..16488ae0393 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/CopyTreeVisitorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/CopyTreeVisitorTests.cs @@ -53,6 +53,7 @@ public void copies_state_between_dbs(int fullPruningMemoryBudgetMb, int maxDegre clonedDb.Values.Should().BeEquivalentTo(values); clonedDb.KeyWasWrittenWithFlags(keys[0], WriteFlags.LowPriority); + trieDb.KeyWasReadWithFlags(keys[0], ReadFlags.SkipDuplicateRead | ReadFlags.HintCacheMiss); } [Test, Timeout(Timeout.MaxTestTime)] diff --git a/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs index 3e488861d40..59004e4b854 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/KnownChainSizesTests.cs @@ -16,7 +16,7 @@ public void Update_known_chain_sizes() // Pruning size have to be updated frequently ChainSizes.CreateChainSizeInfo(BlockchainIds.Mainnet).PruningSize.Should().BeLessThan(210.GB()); ChainSizes.CreateChainSizeInfo(BlockchainIds.Goerli).PruningSize.Should().BeLessThan(90.GB()); - ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(14.GB()); + ChainSizes.CreateChainSizeInfo(BlockchainIds.Sepolia).PruningSize.Should().BeLessThan(15.GB()); ChainSizes.CreateChainSizeInfo(BlockchainIds.Chiado).PruningSize.Should().Be(null); ChainSizes.CreateChainSizeInfo(BlockchainIds.Gnosis).PruningSize.Should().Be(null); diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.BaseFee.cs b/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.BaseFee.cs index 4e66e5bd708..988136b6896 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.BaseFee.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Producers/BlockProducerBaseTests.BaseFee.cs @@ -255,7 +255,7 @@ public async Task BlockProducer_returns_correct_fork_base_fee() .WithEip1559TransitionBlock(7) .CreateTestBlockchain() .BlocksBeforeTransitionShouldHaveZeroBaseFee() - .AssertNewBlock(Eip1559Constants.ForkBaseFee); + .AssertNewBlock(Eip1559Constants.DefaultForkBaseFee); await scenario.Finish(); } @@ -266,7 +266,7 @@ public async Task BlockProducer_returns_correctly_decreases_base_fee_on_empty_bl .WithEip1559TransitionBlock(6) .CreateTestBlockchain() .BlocksBeforeTransitionShouldHaveZeroBaseFee() - .AssertNewBlock(Eip1559Constants.ForkBaseFee) + .AssertNewBlock(Eip1559Constants.DefaultForkBaseFee) .AssertNewBlock(875000000) .AssertNewBlock(765625000) .AssertNewBlock(669921875) @@ -283,7 +283,7 @@ public async Task BaseFee_should_decrease_when_we_send_transactions_below_gas_ta .CreateTestBlockchain(gasLimit) .DeployContract() .BlocksBeforeTransitionShouldHaveZeroBaseFee() - .AssertNewBlock(Eip1559Constants.ForkBaseFee) + .AssertNewBlock(Eip1559Constants.DefaultForkBaseFee) .SendLegacyTransaction(gasLimit / 3, 20.GWei()) .SendEip1559Transaction(gasLimit / 3, 1.GWei(), 20.GWei()) .AssertNewBlock(875000000) @@ -301,7 +301,7 @@ public async Task BaseFee_should_not_change_when_we_send_transactions_equal_gas_ .CreateTestBlockchain(gasTarget) .DeployContract() .BlocksBeforeTransitionShouldHaveZeroBaseFee() - .AssertNewBlock(Eip1559Constants.ForkBaseFee) + .AssertNewBlock(Eip1559Constants.DefaultForkBaseFee) .SendLegacyTransaction(gasTarget / 2, 20.GWei()) .SendEip1559Transaction(gasTarget / 2, 1.GWei(), 20.GWei()) .AssertNewBlock(875000000) @@ -319,7 +319,7 @@ public async Task BaseFee_should_increase_when_we_send_transactions_above_gas_ta .CreateTestBlockchain(gasTarget) .DeployContract() .BlocksBeforeTransitionShouldHaveZeroBaseFee() - .AssertNewBlock(Eip1559Constants.ForkBaseFee) + .AssertNewBlock(Eip1559Constants.DefaultForkBaseFee) .SendLegacyTransaction(gasTarget / 2, 20.GWei()) .SendEip1559Transaction(gasTarget / 2, 1.GWei(), 20.GWei()) .SendLegacyTransaction(gasTarget / 2, 20.GWei()) @@ -337,7 +337,7 @@ public async Task When_base_fee_decreases_previously_fee_too_low_transaction_is_ .WithEip1559TransitionBlock(6) .CreateTestBlockchain(gasTarget) .BlocksBeforeTransitionShouldHaveZeroBaseFee() - .AssertNewBlock(Eip1559Constants.ForkBaseFee) + .AssertNewBlock(Eip1559Constants.DefaultForkBaseFee) .SendLegacyTransaction(gasTarget / 2, 7.GWei() / 10, nonce: UInt256.Zero) .AssertNewBlock(875000000) .AssertNewBlock(765625000) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs index aa0c2f505e0..6269c21cbb7 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs @@ -290,17 +290,22 @@ public void When_TxLookupLimitIs_NegativeOne_DoNotIndexTxHash() _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[receipts[0].TxHash!.Bytes].Should().BeNull(); } - [Test] - public void Should_not_index_tx_hash_if_blockNumber_is_negative() + [TestCase(1L, false)] + [TestCase(10L, false)] + [TestCase(11L, true)] + public void Should_only_prune_index_tx_hashes_if_blockNumber_is_bigger_than_lookupLimit(long blockNumber, bool WillPruneOldIndicies) { _receiptConfig.TxLookupLimit = 10; CreateStorage(); _blockTree.BlockAddedToMain += - Raise.EventWith(new BlockReplacementEventArgs(Build.A.Block.WithNumber(1).TestObject)); + Raise.EventWith(new BlockReplacementEventArgs(Build.A.Block.WithNumber(blockNumber).TestObject)); Thread.Sleep(100); IEnumerable calls = _blockTree.ReceivedCalls() - .Where(call => !call.GetMethodInfo().Name.EndsWith(nameof(_blockTree.BlockAddedToMain))); - calls.Should().BeEmpty(); + .Where(call => call.GetMethodInfo().Name.EndsWith(nameof(_blockTree.FindBlock))); + if (WillPruneOldIndicies) + calls.Should().NotBeEmpty(); + else + calls.Should().BeEmpty(); } [Test] @@ -342,27 +347,49 @@ public void When_NewHeadBlock_Remove_TxIndex_OfRemovedBlock() [Test] public async Task When_NewHeadBlock_Remove_TxIndex_OfRemovedBlock_Unless_ItsAlsoInNewBlock() { + _receiptConfig.CompactTxIndex = _useCompactReceipts; CreateStorage(); (Block block, TxReceipt[] receipts) = InsertBlock(); + Block block2 = Build.A.Block + .WithParent(block) + .WithNumber(2) + .WithTransactions(Build.A.Transaction.SignedAndResolved(TestItem.PrivateKeyC).TestObject) + .TestObject; + _blockTree.FindBestSuggestedHeader().Returns(block2.Header); + _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(block2)); if (_receiptConfig.CompactTxIndex) { - _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[receipts[0].TxHash!.Bytes].Should().BeEquivalentTo(Rlp.Encode(block.Number).Bytes); + _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[block.Transactions[0].Hash!.Bytes].Should().BeEquivalentTo(Rlp.Encode(block.Number).Bytes); } else { - _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[receipts[0].TxHash!.Bytes].Should().NotBeNull(); + _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[block.Transactions[0].Hash!.Bytes].Should().BeEquivalentTo(block.Hash!.Bytes.ToArray()); } - Block newHead = Build.A.Block + Block block3 = Build.A.Block .WithNumber(1) + .WithTransactions(block2.Transactions) + .WithExtraData(new byte[1]) + .TestObject; + Block block4 = Build.A.Block + .WithNumber(2) .WithTransactions(block.Transactions) + .WithExtraData(new byte[1]) .TestObject; - _blockTree.FindBestSuggestedHeader().Returns(newHead.Header); - _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(newHead, block)); + _blockTree.FindBestSuggestedHeader().Returns(block4.Header); + _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(block3, block)); + _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(block4, block2)); await Task.Delay(100); - _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[receipts[0].TxHash!.Bytes].Should().NotBeNull(); + if (_receiptConfig.CompactTxIndex) + { + _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[block4.Transactions[0].Hash!.Bytes].Should().BeEquivalentTo(Rlp.Encode(block4.Number).Bytes); + } + else + { + _receiptsDb.GetColumnDb(ReceiptsColumns.Transactions)[block4.Transactions[0].Hash!.Bytes].Should().BeEquivalentTo(block4.Hash!.Bytes.ToArray()); + } } [Test] diff --git a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs index 353849ad43b..4a7eefa42fb 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockTree.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockTree.cs @@ -1272,7 +1272,7 @@ private ChainLevelInfo UpdateOrCreateLevel(long number, Hash256 hash, BlockInfo /// private bool ShouldCache(long number) { - return number == 0L || Head is null || number <= Head.Number + 1; + return number == 0L || Head is null || number >= Head.Number - HeaderStore.CacheSize; } public ChainLevelInfo? FindLevel(long number) @@ -1434,6 +1434,7 @@ void SetTotalDifficultyDeep(BlockHeader current) { current.TotalDifficulty = current.Difficulty; BlockInfo blockInfo = new(current.Hash, current.Difficulty); + blockInfo.WasProcessed = true; UpdateOrCreateLevel(current.Number, current.Hash, blockInfo); } diff --git a/src/Nethermind/Nethermind.Blockchain/ExitOnBlockNumberHandler.cs b/src/Nethermind/Nethermind.Blockchain/ExitOnBlockNumberHandler.cs new file mode 100644 index 00000000000..472d6e27c67 --- /dev/null +++ b/src/Nethermind/Nethermind.Blockchain/ExitOnBlockNumberHandler.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Config; +using Nethermind.Logging; + +namespace Nethermind.Blockchain; + +/// +/// Just call exit if blocktree report added block is more than a specific number. +/// +public class ExitOnBlockNumberHandler +{ + public ExitOnBlockNumberHandler( + IBlockTree blockTree, + IProcessExitSource processExitSource, + long initConfigExitOnBlockNumber, + ILogManager logManager) + { + ILogger logger = logManager.GetClassLogger(); + + blockTree.BlockAddedToMain += (sender, args) => + { + if (args.Block.Number >= initConfigExitOnBlockNumber) + { + logger.Info($"Block {args.Block.Number} reached. Exiting."); + processExitSource.Exit(0); + } + }; + } +} diff --git a/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs b/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs index ee85beb5d79..cd411837ed0 100644 --- a/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs +++ b/src/Nethermind/Nethermind.Blockchain/FullPruning/CopyTreeVisitor.cs @@ -43,6 +43,9 @@ public CopyTreeVisitor( } public bool IsFullDbScan => true; + + public ReadFlags ExtraReadFlag => ReadFlags.SkipDuplicateRead; + public bool ShouldVisit(Hash256 nextNode) => !_cancellationToken.IsCancellationRequested; public void VisitTree(Hash256 rootHash, TrieVisitContext trieVisitContext) diff --git a/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs b/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs index f5f37cc46b2..87eefab12e8 100644 --- a/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/Headers/HeaderStore.cs @@ -15,8 +15,8 @@ namespace Nethermind.Blockchain.Headers; public class HeaderStore : IHeaderStore { - // SyncProgressResolver MaxLookupBack is 128, add 16 wiggle room - private const int CacheSize = 128 + 16; + // SyncProgressResolver MaxLookupBack is 256, add 16 wiggle room + public const int CacheSize = 256 + 16; private readonly IDb _headerDb; private readonly IDb _blockNumberDb; diff --git a/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs b/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs index d6d35d8b0ef..4d6b55283d9 100644 --- a/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs +++ b/src/Nethermind/Nethermind.Blockchain/ReceiptCanonicalityMonitor.cs @@ -16,24 +16,20 @@ public interface IReceiptMonitor : IDisposable public class ReceiptCanonicalityMonitor : IReceiptMonitor { - private readonly IBlockTree _blockTree; private readonly IReceiptStorage _receiptStorage; private readonly ILogger _logger; public event EventHandler? ReceiptsInserted; - public ReceiptCanonicalityMonitor(IBlockTree? blockTree, IReceiptStorage? receiptStorage, ILogManager? logManager) + public ReceiptCanonicalityMonitor(IReceiptStorage? receiptStorage, ILogManager? logManager) { - _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _blockTree.BlockAddedToMain += OnBlockAddedToMain; + _receiptStorage.ReceiptsInserted += OnBlockAddedToMain; } private void OnBlockAddedToMain(object sender, BlockReplacementEventArgs e) { - _receiptStorage.EnsureCanonical(e.Block); - // we don't want this to be on main processing thread Task.Run(() => TriggerReceiptInsertedEvent(e.Block, e.PreviousBlock)); } @@ -59,7 +55,7 @@ private void TriggerReceiptInsertedEvent(Block newBlock, Block? previousBlock) public void Dispose() { - _blockTree.BlockAddedToMain -= OnBlockAddedToMain; + _receiptStorage.ReceiptsInserted -= OnBlockAddedToMain; } } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs index f1abcef89c9..236540cbe48 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/IReceiptStorage.cs @@ -3,6 +3,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; +using System; namespace Nethermind.Blockchain.Receipts { @@ -14,5 +15,10 @@ public interface IReceiptStorage : IReceiptFinder long MigratedBlockNumber { get; set; } bool HasBlock(long blockNumber, Hash256 hash); void EnsureCanonical(Block block); + + /// + /// Receipts for a block are inserted + /// + event EventHandler ReceiptsInserted; } } diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs index a118a0aec87..4b78996a0ac 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/InMemoryReceiptStorage.cs @@ -11,13 +11,27 @@ namespace Nethermind.Blockchain.Receipts public class InMemoryReceiptStorage : IReceiptStorage { private readonly bool _allowReceiptIterator; + private readonly IBlockTree? _blockTree; private readonly ConcurrentDictionary _receipts = new(); private readonly ConcurrentDictionary _transactions = new(); - public InMemoryReceiptStorage(bool allowReceiptIterator = true) +#pragma warning disable CS0067 + public event EventHandler ReceiptsInserted; +#pragma warning restore CS0067 + + public InMemoryReceiptStorage(bool allowReceiptIterator = true, IBlockTree? blockTree = null) { _allowReceiptIterator = allowReceiptIterator; + _blockTree = blockTree; + if (_blockTree is not null) + _blockTree.BlockAddedToMain += BlockTree_BlockAddedToMain; + } + + private void BlockTree_BlockAddedToMain(object? sender, BlockReplacementEventArgs e) + { + EnsureCanonical(e.Block); + ReceiptsInserted?.Invoke(this, e); } public Hash256 FindBlockHash(Hash256 txHash) diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs index 22f6b7fdc21..802c8d858e5 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/NullReceiptStorage.cs @@ -11,6 +11,10 @@ public class NullReceiptStorage : IReceiptStorage { public static NullReceiptStorage Instance { get; } = new(); +#pragma warning disable CS0067 + public event EventHandler ReceiptsInserted; +#pragma warning restore CS0067 + public Hash256? FindBlockHash(Hash256 hash) => null; private NullReceiptStorage() diff --git a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs index ca972f1aa9a..a186d13d1d2 100644 --- a/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs +++ b/src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs @@ -39,6 +39,8 @@ public class PersistentReceiptStorage : IReceiptStorage private const int CacheSize = 64; private readonly LruCache _receiptsCache = new(CacheSize, CacheSize, "receipts"); + public event EventHandler ReceiptsInserted; + public PersistentReceiptStorage( IColumnsDb receiptsDb, ISpecProvider specProvider, @@ -78,6 +80,8 @@ private void BlockTreeOnBlockAddedToMain(object? sender, BlockReplacementEventAr { RemoveBlockTx(e.PreviousBlock, e.Block); } + EnsureCanonical(e.Block); + ReceiptsInserted?.Invoke(this, e); // Dont block main loop Task.Run(() => diff --git a/src/Nethermind/Nethermind.Config.Test/ConfigFileTestsBase.cs b/src/Nethermind/Nethermind.Config.Test/ConfigFileTestsBase.cs index 315960d295e..d68760d11e3 100644 --- a/src/Nethermind/Nethermind.Config.Test/ConfigFileTestsBase.cs +++ b/src/Nethermind/Nethermind.Config.Test/ConfigFileTestsBase.cs @@ -81,9 +81,9 @@ protected IEnumerable ChiadoConfigs protected IEnumerable GoerliConfigs => Configs.Where(config => config.Contains("goerli")); - [ConfigFileGroup("kovan")] - protected IEnumerable KovanConfigs - => Configs.Where(config => config.Contains("kovan")); + [ConfigFileGroup("holesky")] + protected IEnumerable HoleskyConfigs + => Configs.Where(config => config.Contains("holesky")); [ConfigFileGroup("spaceneth")] protected IEnumerable SpacenethConfigs @@ -103,8 +103,7 @@ protected IEnumerable AuraConfigs .Union(GnosisConfigs) .Union(ChiadoConfigs) .Union(VoltaConfigs) - .Union(EnergyConfigs) - .Union(KovanConfigs); + .Union(EnergyConfigs); [ConfigFileGroup("aura_non_validating")] protected IEnumerable AuraNonValidatingConfigs diff --git a/src/Nethermind/Nethermind.Consensus.Test/TargetAdjustedGasLimitCalculatorTests.cs b/src/Nethermind/Nethermind.Consensus.Test/TargetAdjustedGasLimitCalculatorTests.cs index d3006e499c7..2f96501987d 100644 --- a/src/Nethermind/Nethermind.Consensus.Test/TargetAdjustedGasLimitCalculatorTests.cs +++ b/src/Nethermind/Nethermind.Consensus.Test/TargetAdjustedGasLimitCalculatorTests.cs @@ -27,7 +27,7 @@ public void Is_bump_on_1559_eip_block() TargetAdjustedGasLimitCalculator targetedAdjustedGasLimitCalculator = new(specProvider, new BlocksConfig()); BlockHeader header = Build.A.BlockHeader.WithNumber(londonBlock - 1).WithGasLimit(gasLimit).TestObject; long actualValue = targetedAdjustedGasLimitCalculator.GetGasLimit(header); - Assert.That(actualValue, Is.EqualTo(gasLimit * Eip1559Constants.ElasticityMultiplier)); + Assert.That(actualValue, Is.EqualTo(gasLimit * Eip1559Constants.DefaultElasticityMultiplier)); } } } diff --git a/src/Nethermind/Nethermind.Consensus/Eip1559GasLimitAdjuster.cs b/src/Nethermind/Nethermind.Consensus/Eip1559GasLimitAdjuster.cs index 3c1ecb4138f..98e15179438 100644 --- a/src/Nethermind/Nethermind.Consensus/Eip1559GasLimitAdjuster.cs +++ b/src/Nethermind/Nethermind.Consensus/Eip1559GasLimitAdjuster.cs @@ -14,7 +14,7 @@ public static long AdjustGasLimit(IReleaseSpec releaseSpec, long gasLimit, long long adjustedGasLimit = gasLimit; if (releaseSpec.Eip1559TransitionBlock == blockNumber) { - adjustedGasLimit *= Eip1559Constants.ElasticityMultiplier; + adjustedGasLimit *= releaseSpec.ElasticityMultiplier; } return adjustedGasLimit; diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs index 232c125c38b..f7f5bd8c803 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs @@ -43,7 +43,7 @@ public partial class BlockProcessor : IBlockProcessor /// We use a single receipt tracer for all blocks. Internally receipt tracer forwards most of the calls /// to any block-specific tracers. /// - private readonly BlockReceiptsTracer _receiptsTracer; + protected BlockReceiptsTracer ReceiptsTracer { get; set; } public BlockProcessor( ISpecProvider? specProvider, @@ -67,7 +67,7 @@ public BlockProcessor( _blockTransactionsExecutor = blockTransactionsExecutor ?? throw new ArgumentNullException(nameof(blockTransactionsExecutor)); _beaconBlockRootHandler = new BeaconBlockRootHandler(); - _receiptsTracer = new BlockReceiptsTracer(); + ReceiptsTracer = new BlockReceiptsTracer(); } public event EventHandler BlockProcessed; @@ -226,13 +226,13 @@ protected virtual TxReceipt[] ProcessBlock( { IReleaseSpec spec = _specProvider.GetSpec(block.Header); - _receiptsTracer.SetOtherTracer(blockTracer); - _receiptsTracer.StartNewBlockTrace(block); + ReceiptsTracer.SetOtherTracer(blockTracer); + ReceiptsTracer.StartNewBlockTrace(block); _beaconBlockRootHandler.ApplyContractStateChanges(block, spec, _stateProvider); _stateProvider.Commit(spec); - TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, options, _receiptsTracer, spec); + TxReceipt[] receipts = _blockTransactionsExecutor.ProcessTransactions(block, options, ReceiptsTracer, spec); if (spec.IsEip4844Enabled) { @@ -242,7 +242,7 @@ protected virtual TxReceipt[] ProcessBlock( block.Header.ReceiptsRoot = receipts.GetReceiptsRoot(spec, block.ReceiptsRoot); ApplyMinerRewards(block, blockTracer, spec); _withdrawalProcessor.ProcessWithdrawals(block, spec); - _receiptsTracer.EndBlockTrace(); + ReceiptsTracer.EndBlockTrace(); _stateProvider.Commit(spec); @@ -260,7 +260,7 @@ protected virtual TxReceipt[] ProcessBlock( // TODO: block processor pipeline private void StoreTxReceipts(Block block, TxReceipt[] txReceipts) { - // Setting canonical is done by ReceiptCanonicalityMonitor on block move to main + // Setting canonical is done when the BlockAddedToMain event is firec _receiptStorage.Insert(block, txReceipts, false); } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs index 5f29f9af04e..ed6d13e5520 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/ProcessingStats.cs @@ -124,7 +124,7 @@ public void UpdateStats(Block? block, IBlockTree blockTreeCtx, int recoveryQueue decimal bps = chunkMicroseconds == 0 ? -1 : chunkBlocks / chunkMicroseconds * 1000 * 1000; decimal chunkMs = (chunkMicroseconds == 0 ? -1 : chunkMicroseconds / 1000); decimal runMs = (runMicroseconds == 0 ? -1 : runMicroseconds / 1000); - string blockGas = Evm.Metrics.BlockMinGasPrice != decimal.MaxValue ? $" Gas gwei: {Evm.Metrics.BlockMinGasPrice:N2} .. {whiteText}{Math.Max(Evm.Metrics.BlockMinGasPrice, Evm.Metrics.BlockEstMedianGasPrice):N2}{resetColor} ({Evm.Metrics.BlockAveGasPrice:N2}) .. {Evm.Metrics.BlockMaxGasPrice:N2}" : ""; + string blockGas = Evm.Metrics.BlockMinGasPrice != float.MaxValue ? $" Gas gwei: {Evm.Metrics.BlockMinGasPrice:N2} .. {whiteText}{Math.Max(Evm.Metrics.BlockMinGasPrice, Evm.Metrics.BlockEstMedianGasPrice):N2}{resetColor} ({Evm.Metrics.BlockAveGasPrice:N2}) .. {Evm.Metrics.BlockMaxGasPrice:N2}" : ""; if (chunkBlocks > 1) { _logger.Info($"Processed {block.Number - chunkBlocks + 1,9}...{block.Number,9} | {chunkMs,9:N2} ms | slot {runMs,7:N0} ms |{blockGas}"); diff --git a/src/Nethermind/Nethermind.Consensus/Processing/RecoverSignature.cs b/src/Nethermind/Nethermind.Consensus/Processing/RecoverSignature.cs index 48249fa7c6d..c9e6d56dcb6 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/RecoverSignature.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/RecoverSignature.cs @@ -45,21 +45,55 @@ public void RecoverData(Block block) // so we assume the rest of txs in the block are already recovered return; - var releaseSpec = _specProvider.GetSpec(block.Header); - Parallel.ForEach( - block.Transactions.Where(tx => tx.IsSigned && tx.SenderAddress is null), + block.Transactions.Where(tx => !tx.IsHashCalculated), blockTransaction => { - _txPool.TryGetPendingTransaction(blockTransaction.Hash, out Transaction? transaction); + blockTransaction.CalculateHashInternal(); + }); - Address sender = transaction?.SenderAddress; - Address blockTransactionAddress = blockTransaction.SenderAddress; + var releaseSpec = _specProvider.GetSpec(block.Header); - blockTransaction.SenderAddress = - sender ?? _ecdsa.RecoverAddress(blockTransaction, !releaseSpec.ValidateChainId); - if (_logger.IsTrace) _logger.Trace($"Recovered {blockTransaction.SenderAddress} sender for {blockTransaction.Hash} (tx pool cached value: {sender}, block transaction address: {blockTransactionAddress})"); - }); + int recoverFromEcdsa = 0; + // Don't access txPool in Parallel loop as increases contention + foreach (Transaction blockTransaction in block.Transactions.Where(tx => tx.IsSigned && tx.SenderAddress is null)) + { + _txPool.TryGetPendingTransaction(blockTransaction.Hash, out Transaction? transaction); + + Address sender = transaction?.SenderAddress; + if (sender != null) + { + blockTransaction.SenderAddress = sender; + + if (_logger.IsTrace) _logger.Trace($"Recovered {blockTransaction.SenderAddress} sender for {blockTransaction.Hash} (tx pool cached value: {sender})"); + } + else + { + recoverFromEcdsa++; + } + } + + if (recoverFromEcdsa >= 4) + { + // Recover ecdsa in Parallel + Parallel.ForEach( + block.Transactions.Where(tx => tx.IsSigned && tx.SenderAddress is null), + blockTransaction => + { + blockTransaction.SenderAddress = _ecdsa.RecoverAddress(blockTransaction, !releaseSpec.ValidateChainId); + + if (_logger.IsTrace) _logger.Trace($"Recovered {blockTransaction.SenderAddress} sender for {blockTransaction.Hash}"); + }); + } + else if (recoverFromEcdsa > 0) + { + foreach (Transaction blockTransaction in block.Transactions.Where(tx => tx.IsSigned && tx.SenderAddress is null)) + { + blockTransaction.SenderAddress = _ecdsa.RecoverAddress(blockTransaction, !releaseSpec.ValidateChainId); + + if (_logger.IsTrace) _logger.Trace($"Recovered {blockTransaction.SenderAddress} sender for {blockTransaction.Hash}"); + } + } } } } diff --git a/src/Nethermind/Nethermind.Core.Test/AddressTests.cs b/src/Nethermind/Nethermind.Core.Test/AddressTests.cs index 93b2274d08e..b286e202187 100644 --- a/src/Nethermind/Nethermind.Core.Test/AddressTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/AddressTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections; +using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; @@ -183,6 +184,26 @@ public void Of_contract(long nonce, string expectedAddress) public bool Is_PointEvaluationPrecompile_properly_activated(IReleaseSpec spec) => Address.FromNumber(0x0a).IsPrecompile(spec); + [TestCase(Address.SystemUserHex, false)] + [TestCase("2" + Address.SystemUserHex, false)] + [TestCase("2" + Address.SystemUserHex, true)] + public void Parse_variable_length(string addressHex, bool allowOverflow) + { + var result = Address.TryParseVariableLength(addressHex, out Address? address, allowOverflow); + result.Should().Be(addressHex.Length <= Address.SystemUserHex.Length || allowOverflow); + if (result) + { + address.Should().Be(Address.SystemUser); + } + } + + [Test] + public void Parse_variable_length_too_short() + { + Address.TryParseVariableLength("1", out Address? address).Should().Be(true); + address.Should().Be(new Address("0000000000000000000000000000000000000001")); + } + public static IEnumerable PointEvaluationPrecompileTestCases { get diff --git a/src/Nethermind/Nethermind.Core.Test/BlockHeaderTests.cs b/src/Nethermind/Nethermind.Core.Test/BlockHeaderTests.cs index b2c59314889..849c25d3364 100644 --- a/src/Nethermind/Nethermind.Core.Test/BlockHeaderTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/BlockHeaderTests.cs @@ -114,10 +114,13 @@ public void Eip_1559_CalculateBaseFee(long gasTarget, long baseFee, long expecte IReleaseSpec releaseSpec = Substitute.For(); releaseSpec.IsEip1559Enabled.Returns(true); releaseSpec.Eip1559BaseFeeMinValue.Returns((UInt256?)minimalBaseFee); + releaseSpec.ForkBaseFee.Returns(Eip1559Constants.DefaultForkBaseFee); + releaseSpec.BaseFeeMaxChangeDenominator.Returns(Eip1559Constants.DefaultBaseFeeMaxChangeDenominator); + releaseSpec.ElasticityMultiplier.Returns(Eip1559Constants.DefaultElasticityMultiplier); BlockHeader blockHeader = Build.A.BlockHeader.TestObject; blockHeader.Number = 2001; - blockHeader.GasLimit = gasTarget * Eip1559Constants.ElasticityMultiplier; + blockHeader.GasLimit = gasTarget * Eip1559Constants.DefaultElasticityMultiplier; blockHeader.BaseFeePerGas = (UInt256)baseFee; blockHeader.GasUsed = gasUsed; UInt256 actualBaseFee = BaseFeeCalculator.Calculate(blockHeader, releaseSpec); @@ -141,10 +144,13 @@ public void Eip_1559_CalculateBaseFee_shared_test_cases((BaseFeeTestCases Info, { IReleaseSpec releaseSpec = Substitute.For(); releaseSpec.IsEip1559Enabled.Returns(true); + releaseSpec.ForkBaseFee.Returns(Eip1559Constants.DefaultForkBaseFee); + releaseSpec.BaseFeeMaxChangeDenominator.Returns(Eip1559Constants.DefaultBaseFeeMaxChangeDenominator); + releaseSpec.ElasticityMultiplier.Returns(Eip1559Constants.DefaultElasticityMultiplier); BlockHeader blockHeader = Build.A.BlockHeader.TestObject; blockHeader.Number = 2001; - blockHeader.GasLimit = testCase.Info.ParentTargetGasUsed * Eip1559Constants.ElasticityMultiplier; + blockHeader.GasLimit = testCase.Info.ParentTargetGasUsed * Eip1559Constants.DefaultElasticityMultiplier; blockHeader.BaseFeePerGas = (UInt256)testCase.Info.ParentBaseFee; blockHeader.GasUsed = testCase.Info.ParentGasUsed; UInt256 actualBaseFee = BaseFeeCalculator.Calculate(blockHeader, releaseSpec); diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index e23c0705e0d..4215fda3fa3 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -160,13 +160,13 @@ protected virtual async Task Build(ISpecProvider? specProvider = _trieStoreWatcher = new TrieStoreBoundaryWatcher(WorldStateManager, BlockTree, LogManager); - ReceiptStorage = new InMemoryReceiptStorage(); + ReceiptStorage = new InMemoryReceiptStorage(blockTree: BlockTree); VirtualMachine virtualMachine = new(new BlockhashProvider(BlockTree, LogManager), SpecProvider, LogManager); TxProcessor = new TransactionProcessor(SpecProvider, State, virtualMachine, LogManager); BlockPreprocessorStep = new RecoverSignatures(EthereumEcdsa, TxPool, SpecProvider, LogManager); HeaderValidator = new HeaderValidator(BlockTree, Always.Valid, SpecProvider, LogManager); - new ReceiptCanonicalityMonitor(BlockTree, ReceiptStorage, LogManager); + new ReceiptCanonicalityMonitor(ReceiptStorage, LogManager); BlockValidator = new BlockValidator( new TxValidator(SpecProvider.ChainId), @@ -283,7 +283,7 @@ protected virtual TxPool.TxPool CreateTxPool() => EthereumEcdsa, new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(SpecProvider), BlockTree, ReadOnlyState), - new TxPoolConfig() { BlobSupportEnabled = true }, + new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory }, new TxValidator(SpecProvider.ChainId), LogManager, TransactionComparerProvider.GetDefaultComparer()); diff --git a/src/Nethermind/Nethermind.Core.Test/Json/DoubleArrayConverterTests.cs b/src/Nethermind/Nethermind.Core.Test/Json/DoubleArrayConverterTests.cs new file mode 100644 index 00000000000..d177b45f40d --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/Json/DoubleArrayConverterTests.cs @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Text.Json; +using Nethermind.Serialization.Json; + +using NUnit.Framework; + +namespace Nethermind.Core.Test.Json; + +[TestFixture] +public class DoubleArrayConverterTests : ConverterTestBase +{ + static readonly DoubleArrayConverter converter = new(); + + [Test] + public void Test_roundtrip() + { + TestConverter(new double[] { -0.5, 0.5, 1.0, 1.5, 2.0, 2.5 }, (a, b) => a.AsSpan().SequenceEqual(b), converter); + TestConverter(new double[] { 1, 1, 1, 1 }, (a, b) => a.AsSpan().SequenceEqual(b), converter); + TestConverter(new double[] { 0, 0, 0, 0 }, (a, b) => a.AsSpan().SequenceEqual(b), converter); + TestConverter(Array.Empty(), (a, b) => a.AsSpan().SequenceEqual(b), converter); + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/MCSLockTests.cs b/src/Nethermind/Nethermind.Core.Test/MCSLockTests.cs new file mode 100644 index 00000000000..889643fce1b --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/MCSLockTests.cs @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Threading; +using NUnit.Framework; + +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace Nethermind.Core.Test; + +[TestFixture] +public class MCSLockTests +{ + private McsLock mcsLock; + + [SetUp] + public void Setup() + { + mcsLock = new McsLock(); + } + + [Test] + public void SingleThreadAcquireRelease() + { + using (var handle = mcsLock.Acquire()) + { + Thread.Sleep(10); + } + + Assert.Pass(); // Test passes if no deadlock or exception occurs. + } + + [Test] + public void MultipleThreads() + { + int counter = 0; + int numberOfThreads = 10; + var threads = new List(); + + for (int i = 0; i < numberOfThreads; i++) + { + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + + counter++; + }); + threads.Add(thread); + thread.Start(); + } + + foreach (Thread thread in threads) + { + thread.Join(); // Wait for all threads to complete. + } + + Assert.That(counter, Is.EqualTo(numberOfThreads)); // Counter should equal the number of threads. + } + + [Test] + public void LockFairnessTest() + { + int numberOfThreads = 10; + var executionOrder = new List(); + var threads = new List(); + + for (int i = 0; i < numberOfThreads; i++) + { + int threadId = i; + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + executionOrder.Add(threadId); + Thread.Sleep(15); // Ensure the order is maintained + }); + threads.Add(thread); + thread.Start(); + Thread.Sleep(1); // Ensure the order is maintained + } + + foreach (Thread thread in threads) + { + thread.Join(); + } + + var expectedOrder = Enumerable.Range(0, numberOfThreads).ToList(); + CollectionAssert.AreEqual(expectedOrder, executionOrder, "Threads did not acquire lock in the order they were started."); + } + + [Test] + public void NonReentrantTest() + { + bool reentrancyDetected = false; + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + try + { + using var innerHandle = mcsLock.Acquire(); // Attempt to re-lock + } + catch + { + reentrancyDetected = true; + } + }); + + thread.Start(); + thread.Join(); + + Assert.IsTrue(reentrancyDetected, "Reentrancy was not properly detected."); + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/McsPriorityLock.cs b/src/Nethermind/Nethermind.Core.Test/McsPriorityLock.cs new file mode 100644 index 00000000000..0eabe4c1eb5 --- /dev/null +++ b/src/Nethermind/Nethermind.Core.Test/McsPriorityLock.cs @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Threading; +using NUnit.Framework; +using NUnit.Framework.Internal; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace Nethermind.Core.Test; + +[TestFixture] +public class McsPriorityLockTests +{ + private McsPriorityLock mcsLock; + + [SetUp] + public void Setup() + { + mcsLock = new McsPriorityLock(); + } + + [Test] + public void SingleThreadAcquireRelease() + { + using (var handle = mcsLock.Acquire()) + { + Thread.Sleep(10); + } + + Assert.Pass(); // Test passes if no deadlock or exception occurs. + } + + [Test] + public void MultipleThreads() + { + int counter = 0; + int numberOfThreads = 10; + var threads = new List(); + + for (int i = 0; i < numberOfThreads; i++) + { + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + + counter++; + }); + threads.Add(thread); + thread.Start(); + } + + foreach (Thread thread in threads) + { + thread.Join(); // Wait for all threads to complete. + } + + Assert.That(counter, Is.EqualTo(numberOfThreads)); // Counter should equal the number of threads. + } + + [Test] + public void LockFairnessTest() + { + int numberOfThreads = 10; + var executionOrder = new List(); + var threads = new List(); + + for (int i = 0; i < numberOfThreads; i++) + { + int threadId = i; + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + executionOrder.Add(threadId); + Thread.Sleep(15); // Ensure the order is maintained + }); + threads.Add(thread); + thread.Start(); + Thread.Sleep(1); // Ensure the order is maintained + } + + foreach (Thread thread in threads) + { + thread.Join(); + } + + var expectedOrder = Enumerable.Range(0, numberOfThreads).ToList(); + CollectionAssert.AreEqual(expectedOrder, executionOrder, "Threads did not acquire lock in the order they were started."); + } + + + [Test] + public void PriorityQueueJumpingTest() + { + int numberOfThreads = 100; + var threads = new List(); + List executionOrder = new(); + Dictionary threadPriorities = new(); + + // Create threads with varying priorities. + for (int i = 0; i < numberOfThreads; i++) + { + ThreadPriority priority = i % 2 == 0 ? ThreadPriority.Highest : ThreadPriority.Normal; // Alternate priorities + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + executionOrder.Add(Thread.CurrentThread.ManagedThreadId); + Thread.Sleep(25); // Simulate work + }); + thread.Priority = priority; // Set thread priority + threads.Add(thread); + threadPriorities[thread] = priority; + } + + // Start threads. + foreach (var thread in threads) + { + thread.Start(); + } + + // Wait for all threads to complete. + foreach (var thread in threads) + { + thread.Join(); + } + + // Analyze execution order based on priority. + int lowPriorityFirst = 0; + for (int i = 0; i < executionOrder.Count - 1; i++) + { + int currentThreadId = executionOrder[i]; + int nextThreadId = executionOrder[i + 1]; + Thread currentThread = threads.First(t => t.ManagedThreadId == currentThreadId); + Thread nextThread = threads.First(t => t.ManagedThreadId == nextThreadId); + + if (threadPriorities[currentThread] < threadPriorities[nextThread]) + { + lowPriorityFirst++; + } + } + + // Some lower priority threads will acquire first; we are asserting that they mostly queue jump + Assert.That(lowPriorityFirst < (numberOfThreads / 8), Is.True, "High priority threads did not acquire the lock before lower priority ones."); + } + + [Test] + public void NonReentrantTest() + { + bool reentrancyDetected = false; + var thread = new Thread(() => + { + using var handle = mcsLock.Acquire(); + try + { + using var innerHandle = mcsLock.Acquire(); // Attempt to re-lock + } + catch + { + reentrancyDetected = true; + } + }); + + thread.Start(); + thread.Join(); + + Assert.IsTrue(reentrancyDetected, "Reentrancy was not properly detected."); + } +} diff --git a/src/Nethermind/Nethermind.Core.Test/RateLimiterTests.cs b/src/Nethermind/Nethermind.Core.Test/RateLimiterTests.cs index 2db52069ae4..84d4f1a653d 100644 --- a/src/Nethermind/Nethermind.Core.Test/RateLimiterTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/RateLimiterTests.cs @@ -48,7 +48,7 @@ public async Task RateLimiter_should_throw_when_cancelled() RateLimiter rateLimiter = new(1); await rateLimiter.WaitAsync(CancellationToken.None); CancellationTokenSource cts = new(); - ValueTask waitTask = rateLimiter.WaitAsync(cts.Token); + Task waitTask = rateLimiter.WaitAsync(cts.Token); cts.Cancel(); Func act = async () => await waitTask; diff --git a/src/Nethermind/Nethermind.Core/Address.cs b/src/Nethermind/Nethermind.Core/Address.cs index 3edfee8d79f..98b1e919b72 100644 --- a/src/Nethermind/Nethermind.Core/Address.cs +++ b/src/Nethermind/Nethermind.Core/Address.cs @@ -24,7 +24,8 @@ public class Address : IEquatable
, IComparable
private const int PrefixedHexCharsCount = 2 + HexCharsCount; // 0x5a4eab120fb44eb6684e5e32785702ff45ea344d public static Address Zero { get; } = new(new byte[Size]); - public static Address SystemUser { get; } = new("0xfffffffffffffffffffffffffffffffffffffffe"); + public const string SystemUserHex = "0xfffffffffffffffffffffffffffffffffffffffe"; + public static Address SystemUser { get; } = new(SystemUserHex); public byte[] Bytes { get; } @@ -91,18 +92,31 @@ public static bool TryParse(string? value, out Address? address) /// /// Parses string value to Address. String can be shorter than 20 bytes long, it is padded with leading 0's then. /// - public static bool TryParseVariableLength(string? value, out Address? address) + public static bool TryParseVariableLength(string? value, out Address? address, bool allowOverflow = false) { if (value is not null) { - try + const int size = Size << 1; + + int start = value is ['0', 'x', ..] ? 2 : 0; + ReadOnlySpan span = value.AsSpan(start); + if (span.Length > size) { - address = new Address(Extensions.Bytes.FromHexString(value, Size)); - return true; + if (allowOverflow) + { + span = span.Slice(value.Length - size); + } + else + { + goto False; + } } - catch (IndexOutOfRangeException) { } + + address = new Address(Extensions.Bytes.FromHexString(span, Size)); + return true; } + False: address = default; return false; } diff --git a/src/Nethermind/Nethermind.Core/BaseFeeCalculator.cs b/src/Nethermind/Nethermind.Core/BaseFeeCalculator.cs index 70b72473a73..268997f6df9 100644 --- a/src/Nethermind/Nethermind.Core/BaseFeeCalculator.cs +++ b/src/Nethermind/Nethermind.Core/BaseFeeCalculator.cs @@ -18,7 +18,7 @@ public static UInt256 Calculate(BlockHeader parent, IEip1559Spec specFor1559) long gasDelta; UInt256 feeDelta; bool isForkBlockNumber = specFor1559.Eip1559TransitionBlock == parent.Number + 1; - long parentGasTarget = parent.GasLimit / Eip1559Constants.ElasticityMultiplier; + long parentGasTarget = parent.GasLimit / specFor1559.ElasticityMultiplier; if (isForkBlockNumber) parentGasTarget = parent.GasLimit; @@ -30,20 +30,20 @@ public static UInt256 Calculate(BlockHeader parent, IEip1559Spec specFor1559) { gasDelta = parent.GasUsed - parentGasTarget; feeDelta = UInt256.Max( - parentBaseFee * (UInt256)gasDelta / (UInt256)parentGasTarget / Eip1559Constants.BaseFeeMaxChangeDenominator, + parentBaseFee * (UInt256)gasDelta / (UInt256)parentGasTarget / specFor1559.BaseFeeMaxChangeDenominator, UInt256.One); expectedBaseFee = parentBaseFee + feeDelta; } else { gasDelta = parentGasTarget - parent.GasUsed; - feeDelta = parentBaseFee * (UInt256)gasDelta / (UInt256)parentGasTarget / Eip1559Constants.BaseFeeMaxChangeDenominator; + feeDelta = parentBaseFee * (UInt256)gasDelta / (UInt256)parentGasTarget / specFor1559.BaseFeeMaxChangeDenominator; expectedBaseFee = UInt256.Max(parentBaseFee - feeDelta, 0); } if (isForkBlockNumber) { - expectedBaseFee = Eip1559Constants.ForkBaseFee; + expectedBaseFee = specFor1559.ForkBaseFee; } if (specFor1559.Eip1559BaseFeeMinValue.HasValue) diff --git a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs index c6c71525a5d..3a15a64b2dc 100644 --- a/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs +++ b/src/Nethermind/Nethermind.Core/Buffers/CappedArray.cs @@ -2,8 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; namespace Nethermind.Core.Buffers; @@ -13,10 +11,10 @@ namespace Nethermind.Core.Buffers; /// underlying array can be null and this struct is meant to be non nullable, checking the `IsNull` property to check /// if it represent null. /// -public struct CappedArray +public readonly struct CappedArray { private readonly T[]? _array = null; - private int _length = 0; + private readonly int _length = 0; public CappedArray(T[]? array, int length) { @@ -44,11 +42,7 @@ public static implicit operator CappedArray(T[]? array) return new CappedArray(array); } - public int Length - { - readonly get => _length; - set => _length = value; - } + public readonly int Length => _length; public readonly T[]? Array => _array; public readonly bool IsUncapped => _length == _array?.Length; @@ -60,7 +54,7 @@ public readonly Span AsSpan() return _array.AsSpan()[..Length]; } - public T[]? ToArray() + public readonly T[]? ToArray() { if (_array is null) return null; if (_length == _array?.Length) return _array; diff --git a/src/Nethermind/Nethermind.Core/Caching/LruCache.cs b/src/Nethermind/Nethermind.Core/Caching/LruCache.cs index 6301cd72da5..d96a4bc8d4b 100644 --- a/src/Nethermind/Nethermind.Core/Caching/LruCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/LruCache.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; using Nethermind.Core.Extensions; +using Nethermind.Core.Threading; namespace Nethermind.Core.Caching { @@ -13,6 +13,7 @@ public sealed class LruCache : ICache where TKey : n { private readonly int _maxCapacity; private readonly Dictionary> _cacheMap; + private readonly McsLock _lock = new(); private LinkedListNode? _leastRecentlyUsed; public LruCache(int maxCapacity, int startCapacity, string name) @@ -30,16 +31,18 @@ public LruCache(int maxCapacity, string name) { } - [MethodImpl(MethodImplOptions.Synchronized)] public void Clear() { + using var lockRelease = _lock.Acquire(); + _leastRecentlyUsed = null; _cacheMap.Clear(); } - [MethodImpl(MethodImplOptions.Synchronized)] public TValue Get(TKey key) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { TValue value = node.Value.Value; @@ -53,9 +56,10 @@ public TValue Get(TKey key) #pragma warning restore 8603 } - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryGet(TKey key, out TValue value) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { value = node.Value.Value; @@ -70,12 +74,13 @@ public bool TryGet(TKey key, out TValue value) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] public bool Set(TKey key, TValue val) { + using var lockRelease = _lock.Acquire(); + if (val is null) { - return Delete(key); + return DeleteNoLock(key); } if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) @@ -101,8 +106,14 @@ public bool Set(TKey key, TValue val) } } - [MethodImpl(MethodImplOptions.Synchronized)] public bool Delete(TKey key) + { + using var lockRelease = _lock.Acquire(); + + return DeleteNoLock(key); + } + + private bool DeleteNoLock(TKey key) { if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { @@ -114,12 +125,17 @@ public bool Delete(TKey key) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] - public bool Contains(TKey key) => _cacheMap.ContainsKey(key); + public bool Contains(TKey key) + { + using var lockRelease = _lock.Acquire(); + + return _cacheMap.ContainsKey(key); + } - [MethodImpl(MethodImplOptions.Synchronized)] public KeyValuePair[] ToArray() { + using var lockRelease = _lock.Acquire(); + int i = 0; KeyValuePair[] array = new KeyValuePair[_cacheMap.Count]; foreach (KeyValuePair> kvp in _cacheMap) diff --git a/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs b/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs index 4dd895c9680..cc58f807a16 100644 --- a/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/LruKeyCache.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; using Nethermind.Core.Extensions; +using Nethermind.Core.Threading; namespace Nethermind.Core.Caching { @@ -14,6 +14,7 @@ public sealed class LruKeyCache where TKey : notnull private readonly int _maxCapacity; private readonly string _name; private readonly Dictionary> _cacheMap; + private readonly McsLock _lock = new(); private LinkedListNode? _leastRecentlyUsed; public LruKeyCache(int maxCapacity, int startCapacity, string name) @@ -30,16 +31,18 @@ public LruKeyCache(int maxCapacity, string name) { } - [MethodImpl(MethodImplOptions.Synchronized)] public void Clear() { + using var lockRelease = _lock.Acquire(); + _leastRecentlyUsed = null; _cacheMap.Clear(); } - [MethodImpl(MethodImplOptions.Synchronized)] public bool Get(TKey key) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); @@ -49,9 +52,10 @@ public bool Get(TKey key) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] public bool Set(TKey key) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { LinkedListNode.MoveToMostRecent(ref _leastRecentlyUsed, node); @@ -74,9 +78,10 @@ public bool Set(TKey key) } } - [MethodImpl(MethodImplOptions.Synchronized)] public void Delete(TKey key) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { LinkedListNode.Remove(ref _leastRecentlyUsed, node); diff --git a/src/Nethermind/Nethermind.Core/Caching/SpanLruCache.cs b/src/Nethermind/Nethermind.Core/Caching/SpanLruCache.cs index 11d36c676f4..4436df72a7d 100644 --- a/src/Nethermind/Nethermind.Core/Caching/SpanLruCache.cs +++ b/src/Nethermind/Nethermind.Core/Caching/SpanLruCache.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Runtime.CompilerServices; using Nethermind.Core.Collections; +using Nethermind.Core.Threading; namespace Nethermind.Core.Caching { @@ -19,6 +20,7 @@ public sealed class SpanLruCache : ISpanCache where { private readonly int _maxCapacity; private readonly SpanDictionary> _cacheMap; + private readonly McsLock _lock = new(); private LinkedListNode? _leastRecentlyUsed; public SpanLruCache(int maxCapacity, int startCapacity, string name, ISpanEqualityComparer comparer) @@ -29,16 +31,18 @@ public SpanLruCache(int maxCapacity, int startCapacity, string name, ISpanEquali _cacheMap = new SpanDictionary>(startCapacity, comparer); } - [MethodImpl(MethodImplOptions.Synchronized)] public void Clear() { + using var lockRelease = _lock.Acquire(); + _leastRecentlyUsed = null; _cacheMap.Clear(); } - [MethodImpl(MethodImplOptions.Synchronized)] public TValue Get(ReadOnlySpan key) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { TValue value = node.Value.Value; @@ -52,9 +56,10 @@ public TValue Get(ReadOnlySpan key) #pragma warning restore 8603 } - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryGet(ReadOnlySpan key, out TValue value) { + using var lockRelease = _lock.Acquire(); + if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { value = node.Value.Value; @@ -69,12 +74,13 @@ public bool TryGet(ReadOnlySpan key, out TValue value) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] public bool Set(ReadOnlySpan key, TValue val) { + using var lockRelease = _lock.Acquire(); + if (val is null) { - return Delete(key); + return DeleteNoLock(key); } if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) @@ -101,8 +107,14 @@ public bool Set(ReadOnlySpan key, TValue val) } } - [MethodImpl(MethodImplOptions.Synchronized)] public bool Delete(ReadOnlySpan key) + { + using var lockRelease = _lock.Acquire(); + + return DeleteNoLock(key); + } + + private bool DeleteNoLock(ReadOnlySpan key) { if (_cacheMap.TryGetValue(key, out LinkedListNode? node)) { @@ -114,14 +126,26 @@ public bool Delete(ReadOnlySpan key) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] - public bool Contains(ReadOnlySpan key) => _cacheMap.ContainsKey(key); + public bool Contains(ReadOnlySpan key) + { + using var lockRelease = _lock.Acquire(); - [MethodImpl(MethodImplOptions.Synchronized)] - public IDictionary Clone() => _cacheMap.ToDictionary(i => i.Key, i => i.Value.Value.Value); + return _cacheMap.ContainsKey(key); + } + + public IDictionary Clone() + { + using var lockRelease = _lock.Acquire(); + + return _cacheMap.ToDictionary(i => i.Key, i => i.Value.Value.Value); + } + + public KeyValuePair[] ToArray() + { + using var lockRelease = _lock.Acquire(); - [MethodImpl(MethodImplOptions.Synchronized)] - public KeyValuePair[] ToArray() => _cacheMap.Select(kv => new KeyValuePair(kv.Key, kv.Value.Value.Value)).ToArray(); + return _cacheMap.Select(kv => new KeyValuePair(kv.Key, kv.Value.Value.Value)).ToArray(); + } private void Replace(ReadOnlySpan key, TValue value) { diff --git a/src/Nethermind/Nethermind.Core/Eip1559Constants.cs b/src/Nethermind/Nethermind.Core/Eip1559Constants.cs index c2895bed910..cab44982cae 100644 --- a/src/Nethermind/Nethermind.Core/Eip1559Constants.cs +++ b/src/Nethermind/Nethermind.Core/Eip1559Constants.cs @@ -14,10 +14,5 @@ public class Eip1559Constants public static readonly UInt256 DefaultBaseFeeMaxChangeDenominator = 8; public static readonly int DefaultElasticityMultiplier = 2; - - // The above values are the default ones. However, we're allowing to override it from genesis - public static UInt256 ForkBaseFee { get; set; } = DefaultForkBaseFee; - public static UInt256 BaseFeeMaxChangeDenominator { get; set; } = DefaultBaseFeeMaxChangeDenominator; - public static long ElasticityMultiplier { get; set; } = DefaultElasticityMultiplier; } } diff --git a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs index 7978ad40043..c772a39b766 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/Bytes.cs @@ -1018,7 +1018,7 @@ public static byte[] FromHexString(string hexString, int length) => hexString is null ? throw new ArgumentNullException(nameof(hexString)) : FromHexString(hexString.AsSpan(), length); [DebuggerStepThrough] - private static byte[] FromHexString(ReadOnlySpan hexString, int length) + public static byte[] FromHexString(ReadOnlySpan hexString, int length) { int start = hexString is ['0', 'x', ..] ? 2 : 0; ReadOnlySpan chars = hexString.Slice(start); diff --git a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs index 0e2e34166e1..f205b7cffc1 100644 --- a/src/Nethermind/Nethermind.Core/IKeyValueStore.cs +++ b/src/Nethermind/Nethermind.Core/IKeyValueStore.cs @@ -70,6 +70,9 @@ public enum ReadFlags // Hint that the workload is likely to need the next value in the sequence and should prefetch it. HintReadAhead = 2, + + // Used for full pruning db to skip duplicate read + SkipDuplicateRead = 4, } [Flags] diff --git a/src/Nethermind/Nethermind.Core/RateLimiter.cs b/src/Nethermind/Nethermind.Core/RateLimiter.cs index 7b443f62443..de66ad7139d 100644 --- a/src/Nethermind/Nethermind.Core/RateLimiter.cs +++ b/src/Nethermind/Nethermind.Core/RateLimiter.cs @@ -48,7 +48,7 @@ public bool IsThrottled() return GetCurrentTick() < _nextSlot; } - public async ValueTask WaitAsync(CancellationToken ctx) + public Task WaitAsync(CancellationToken ctx) { long currentNextSlot = _nextSlot; while (true) @@ -64,8 +64,8 @@ public async ValueTask WaitAsync(CancellationToken ctx) long now = GetCurrentTick(); long toWait = currentNextSlot - now; - if (toWait <= 0) return; + if (toWait <= 0) return Task.CompletedTask; - await Task.Delay(TimeSpan.FromMilliseconds(TickToMs(toWait)), ctx); + return Task.Delay(TimeSpan.FromMilliseconds(TickToMs(toWait)), ctx); } } diff --git a/src/Nethermind/Nethermind.Core/Specs/IEip1559Spec.cs b/src/Nethermind/Nethermind.Core/Specs/IEip1559Spec.cs index 33e5723eed3..0d343448551 100644 --- a/src/Nethermind/Nethermind.Core/Specs/IEip1559Spec.cs +++ b/src/Nethermind/Nethermind.Core/Specs/IEip1559Spec.cs @@ -17,5 +17,8 @@ public interface IEip1559Spec public long Eip1559TransitionBlock { get; } public Address? Eip1559FeeCollector => null; public UInt256? Eip1559BaseFeeMinValue => null; + public UInt256 ForkBaseFee { get; } + public UInt256 BaseFeeMaxChangeDenominator { get; } + public long ElasticityMultiplier { get; } } } diff --git a/src/Nethermind/Nethermind.Core/Threading/McsLock.cs b/src/Nethermind/Nethermind.Core/Threading/McsLock.cs new file mode 100644 index 00000000000..84f23f8f542 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Threading/McsLock.cs @@ -0,0 +1,170 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Threading; + +namespace Nethermind.Core.Threading; + +/// +/// MCSLock (Mellor-Crummey and Scott Lock) provides a fair, scalable mutual exclusion lock. +/// This lock is particularly effective in systems with a high number of threads, as it reduces +/// the contention and spinning overhead typical of other spinlocks. It achieves this by forming +/// a queue of waiting threads, ensuring each thread gets the lock in the order it was requested. +/// +public class McsLock +{ + /// + /// Thread-local storage to ensure each thread has its own node instance. + /// + private readonly ThreadLocal _node = new(() => new ThreadNode()); + + /// + /// Points to the last node in the queue (tail). Used to manage the queue of waiting threads. + /// + private volatile ThreadNode? _tail; + + internal volatile Thread? currentLockHolder = null; + + /// + /// Acquires the lock. If the lock is already held, the calling thread is placed into a queue and + /// enters a busy-wait state until the lock becomes available. + /// + public Disposable Acquire() + { + // Check for reentrancy. + if (Thread.CurrentThread == currentLockHolder) + ThrowInvalidOperationException(); + + ThreadNode node = _node.Value!; + node.Locked = true; + + ThreadNode? predecessor = Interlocked.Exchange(ref _tail, node); + if (predecessor is not null) + { + // If there was a previous tail, it means the lock is already held by someone. + // Set this node as the next node of the predecessor. + predecessor.Next = node; + + // Busy-wait (spin) until our 'Locked' flag is set to false by the thread + // that is releasing the lock. + SpinWait sw = default; + // This lock is more scalable than regular locks as each thread + // spins on their own local flag rather than a shared flag for + // lower cpu cache thrashing. Drawback is it is a strict queue and + // the next thread in line may be sleeping when lock is released. + while (node.Locked) + { + if (sw.NextSpinWillYield) + { + // We use Monitor signalling to try to combat additional latency + // that may be introduced by the strict in-order thread queuing + // rather than letting the SpinWait sleep the thread. + lock (node) + { + if (node.Locked) + // Sleep till signal + Monitor.Wait(node); + else + { + // Acquired the lock + break; + } + } + } + else + { + sw.SpinOnce(); + } + } + } + + // Set current lock holder. + currentLockHolder = Thread.CurrentThread; + + return new Disposable(this); + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowInvalidOperationException() + { + throw new InvalidOperationException("Lock is not reentrant"); + } + } + + /// + /// Used to releases the lock. If there are waiting threads in the queue, it passes the lock to the next + /// thread in line. + /// + public readonly struct Disposable : IDisposable + { + readonly McsLock _lock; + + internal Disposable(McsLock @lock) + { + _lock = @lock; + } + + /// + /// Releases the lock. If there are waiting threads in the queue, it passes the lock to the next + /// thread in line. + /// + public void Dispose() + { + ThreadNode node = _lock._node.Value!; + + // If there is no next node, it means this thread might be the last in the queue. + if (node.Next == null) + { + // Attempt to atomically set the tail to null, indicating no thread is waiting. + // If it is still 'node', then there are no other waiting threads. + if (Interlocked.CompareExchange(ref _lock._tail, null, node) == node) + { + // Clear current lock holder. + _lock.currentLockHolder = null; + return; + } + + // If another thread is in the process of enqueuing itself, + // wait until it finishes setting its node as the 'Next' node. + SpinWait sw = default; + while (node.Next == null) + { + sw.SpinOnce(); + } + } + + // Clear current lock holder. + _lock.currentLockHolder = null; + + // Pass the lock to the next thread by setting its 'Locked' flag to false. + node.Next.Locked = false; + + lock (node.Next) + { + // Wake up next node if sleeping + Monitor.Pulse(node.Next); + } + // Remove the reference to the next node + node.Next = null; + } + } + + /// + /// Node class to represent each thread in the MCS lock queue. + /// + private class ThreadNode + { + /// + /// Indicates whether the current thread is waiting for the lock. + /// + public volatile bool Locked = true; + + /// + /// Points to the next node in the queue. + /// + public ThreadNode? Next = null; + } +} diff --git a/src/Nethermind/Nethermind.Core/Threading/McsPriorityLock.cs b/src/Nethermind/Nethermind.Core/Threading/McsPriorityLock.cs new file mode 100644 index 00000000000..72407ed8efb --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Threading/McsPriorityLock.cs @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics; +using System.Threading; + +namespace Nethermind.Core.Threading; + +/// +/// MCSLock (Mellor-Crummey and Scott Lock) provides a fair, scalable mutual exclusion lock. +/// The McsPriorityLock allows higher priority threads to queue jump on the lock queue. +/// This lock is particularly effective in systems with a high number of threads, as it reduces +/// the contention and spinning overhead typical of other spinlocks. It achieves this by forming +/// a queue of waiting threads, ensuring each thread gets the lock in the order it was requested. +/// +public class McsPriorityLock +{ + private readonly int HalfCores = Math.Max(Environment.ProcessorCount / 2, 1); + + private readonly McsLock _coreLock = new(); + private readonly McsLock[] _queuedLocks; + private uint _queueId; + + public McsPriorityLock() + { + var queue = new McsLock[HalfCores]; + for (var i = 0; i < queue.Length; i++) + { + queue[i] = new McsLock(); + } + + _queuedLocks = queue; + } + + /// + /// Acquires the lock. If the lock is already held, the calling thread is placed into a queue and + /// enters a busy-wait state until the lock becomes available. + /// + /// Higher priority threads will queue jump. + /// + public McsLock.Disposable Acquire() + { + // Check for reentrancy. + if (Thread.CurrentThread == _coreLock.currentLockHolder) + ThrowInvalidOperationException(); + + var isPriority = Thread.CurrentThread.Priority > ThreadPriority.Normal; + if (!isPriority) + // If not a priority thread max of half processors can being to acquire the lock (e.g. block processing) + return NonPriorityAcquire(); + + return _coreLock.Acquire(); + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowInvalidOperationException() + { + throw new InvalidOperationException("Lock is not reentrant"); + } + } + + private McsLock.Disposable NonPriorityAcquire() + { + var queueId = Interlocked.Increment(ref _queueId) % (uint)_queuedLocks.Length; + + using var handle = _queuedLocks[queueId].Acquire(); + + return _coreLock.Acquire(); + } +} diff --git a/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs b/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs new file mode 100644 index 00000000000..31d892ba8b6 --- /dev/null +++ b/src/Nethermind/Nethermind.Core/Threading/ThreadExtensions.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading; + +namespace Nethermind.Core.Threading; + +public static class ThreadExtensions +{ + public readonly struct Disposable : IDisposable + { + private readonly Thread _thread; + private readonly ThreadPriority _previousPriority; + + internal Disposable(Thread thread) + { + _thread = thread; + _previousPriority = thread.Priority; + thread.Priority = ThreadPriority.AboveNormal; + } + + public void Dispose() + { + _thread.Priority = _previousPriority; + } + } + + public static Disposable BoostPriority(this Thread thread) + { + return new Disposable(thread); + } +} diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index a8c6956b9df..f83f0ea5d75 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -4,13 +4,16 @@ using System; using System.Buffers; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Text; +using System.Text.Json.Serialization; using Microsoft.Extensions.ObjectPool; using Nethermind.Core.Crypto; using Nethermind.Core.Eip2930; using Nethermind.Core.Extensions; using Nethermind.Int256; +[assembly: InternalsVisibleTo("Nethermind.Consensus")] namespace Nethermind.Core { [DebuggerDisplay("{Hash}, Value: {Value}, To: {To}, Gas: {GasLimit}")] @@ -53,29 +56,45 @@ public class Transaction public bool IsMessageCall => To is not null; private Hash256? _hash; - public Hash256? Hash + + [JsonIgnore] + internal bool IsHashCalculated => _hash is not null; + internal Hash256 CalculateHashInternal() { - get + Hash256? hash = _hash; + if (hash is not null) return hash; + + lock (this) { - if (_hash is not null) return _hash; + hash = _hash; + if (hash is not null) return hash; - lock (this) + if (_preHash.Length > 0) { - if (_hash is not null) return _hash; - - if (_preHash.Length > 0) - { - _hash = Keccak.Compute(_preHash.Span); - ClearPreHashInternal(); - } + _hash = hash = Keccak.Compute(_preHash.Span); + ClearPreHashInternal(); } + } - return _hash; + return hash!; + } + + public Hash256? Hash + { + get + { + Hash256? hash = _hash; + if (hash is not null) return hash; + + return CalculateHashInternal(); } set { - ClearPreHash(); - _hash = value; + lock (this) + { + ClearPreHash(); + _hash = value; + } } } diff --git a/src/Nethermind/Nethermind.Core/TransactionReceipt.cs b/src/Nethermind/Nethermind.Core/TransactionReceipt.cs index ea5f9c57531..5de0b0d85f3 100644 --- a/src/Nethermind/Nethermind.Core/TransactionReceipt.cs +++ b/src/Nethermind/Nethermind.Core/TransactionReceipt.cs @@ -39,6 +39,9 @@ public class TxReceipt public LogEntry[]? Logs { get; set; } public string? Error { get; set; } + public ulong? DepositNonce { get; set; } + public ulong? DepositReceiptVersion { get; set; } + /// /// Ignores receipt output on RLP serialization. /// Output is either StateRoot or StatusCode depending on eip configuration. diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index 63c44fce165..f4601b5f0e4 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -64,6 +64,12 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags return _mainDb.GetAllCore(iterator); } + public IEnumerable GetAllKeys(bool ordered = false) + { + Iterator iterator = _mainDb.CreateIterator(ordered, _columnFamily); + return _mainDb.GetAllKeysCore(iterator); + } + public IEnumerable GetAllValues(bool ordered = false) { Iterator iterator = _mainDb.CreateIterator(ordered, _columnFamily); diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index d17d4179436..99f4b222dcb 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -6,7 +6,6 @@ using System.Collections.Generic; using System.IO; using System.IO.Abstractions; -using System.Net.Sockets; using System.Reflection; using System.Threading; using ConcurrentCollections; @@ -590,6 +589,7 @@ internal Span GetSpanWithColumnFamily(ReadOnlySpan key, ColumnFamily try { Span span = _db.GetSpan(key, cf); + if (!span.IsNullOrEmpty()) { Interlocked.Increment(ref _allocatedSpan); @@ -658,6 +658,17 @@ protected internal Iterator CreateIterator(bool ordered = false, ColumnFamilyHan } } + public IEnumerable GetAllKeys(bool ordered = false) + { + if (_isDisposing) + { + throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}"); + } + + Iterator iterator = CreateIterator(ordered); + return GetAllKeysCore(iterator); + } + public IEnumerable GetAllValues(bool ordered = false) { ObjectDisposedException.ThrowIf(_isDisposing, this); @@ -708,6 +719,48 @@ internal IEnumerable GetAllValuesCore(Iterator iterator) } } + internal IEnumerable GetAllKeysCore(Iterator iterator) + { + try + { + try + { + iterator.SeekToFirst(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + + while (iterator.Valid()) + { + yield return iterator.Key(); + try + { + iterator.Next(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + } + finally + { + try + { + iterator.Dispose(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + } + public IEnumerable> GetAllCore(Iterator iterator) { try @@ -838,6 +891,7 @@ public void Dispose() try { _dbOnTheRocks._db.Write(_rocksBatch, _dbOnTheRocks.WriteFlagsToWriteOptions(_writeFlags)); + _dbOnTheRocks._currentBatches.TryRemove(this); ReturnWriteBatch(_rocksBatch); } diff --git a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs index cc1309f755f..c9adaa5b6a8 100644 --- a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs +++ b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs @@ -77,6 +77,8 @@ public void Clear() { } public IEnumerable> GetAll(bool ordered = false) => _recordDb.GetAll(); + public IEnumerable GetAllKeys(bool ordered = false) => _recordDb.GetAllKeys(); + public IEnumerable GetAllValues(bool ordered = false) => _recordDb.GetAllValues(); public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db.Test/FullPruning/FullPruningDbTests.cs b/src/Nethermind/Nethermind.Db.Test/FullPruning/FullPruningDbTests.cs index 3146ea71d89..18da3dabb9b 100644 --- a/src/Nethermind/Nethermind.Db.Test/FullPruning/FullPruningDbTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/FullPruning/FullPruningDbTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using FluentAssertions; +using Nethermind.Core; using Nethermind.Db.FullPruning; using NSubstitute; using NUnit.Framework; @@ -65,6 +66,34 @@ public void during_pruning_writes_to_both_dbs() test.CurrentMirrorDb[key].Should().BeEquivalentTo(value); } + [Test] + public void during_pruning_duplicate_on_read() + { + TestContext test = new(); + byte[] key = { 1, 2 }; + byte[] value = { 5, 6 }; + test.FullPruningDb[key] = value; + + test.FullPruningDb.TryStartPruning(out IPruningContext _); + + test.FullPruningDb.Get(key); + test.CurrentMirrorDb[key].Should().BeEquivalentTo(value); + } + + [Test] + public void during_pruning_dont_duplicate_read_with_skip_duplicate_read() + { + TestContext test = new(); + byte[] key = { 1, 2 }; + byte[] value = { 5, 6 }; + test.FullPruningDb[key] = value; + + test.FullPruningDb.TryStartPruning(out IPruningContext _); + + test.FullPruningDb.Get(key, ReadFlags.SkipDuplicateRead); + test.CurrentMirrorDb[key].Should().BeNull(); + } + [Test] public void increments_metrics_on_write_to_mirrored_db() { diff --git a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs index 56b4f1d4848..107372461dd 100644 --- a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs +++ b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs @@ -6,5 +6,6 @@ namespace Nethermind.Db; public enum BlobTxsColumns { FullBlobTxs, - LightBlobTxs + LightBlobTxs, + ProcessedTxs } diff --git a/src/Nethermind/Nethermind.Db/CompressingDb.cs b/src/Nethermind/Nethermind.Db/CompressingDb.cs index 8a2dc1d26b9..43bd72d3e98 100644 --- a/src/Nethermind/Nethermind.Db/CompressingDb.cs +++ b/src/Nethermind/Nethermind.Db/CompressingDb.cs @@ -127,6 +127,9 @@ private static ReadOnlySpan Compress(ReadOnlySpan bytes, Span public IEnumerable> GetAll(bool ordered = false) => _wrapped.GetAll(ordered) .Select(kvp => new KeyValuePair(kvp.Key, Decompress(kvp.Value))); + public IEnumerable GetAllKeys(bool ordered = false) => + _wrapped.GetAllKeys(ordered).Select(Decompress); + public IEnumerable GetAllValues(bool ordered = false) => _wrapped.GetAllValues(ordered).Select(Decompress); diff --git a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs index e610a94e49f..20c850ab377 100755 --- a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs +++ b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs @@ -52,7 +52,7 @@ public byte[]? this[ReadOnlySpan key] public byte[]? Get(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { byte[]? value = _currentDb.Get(key, flags); // we are reading from the main DB - if (_pruningContext?.DuplicateReads == true) + if (value != null && _pruningContext?.DuplicateReads == true && (flags & ReadFlags.SkipDuplicateRead) == 0) { Duplicate(_pruningContext.CloningDb, key, value, WriteFlags.None); } @@ -63,7 +63,7 @@ public byte[]? this[ReadOnlySpan key] public Span GetSpan(ReadOnlySpan key, ReadFlags flags = ReadFlags.None) { Span value = _currentDb.GetSpan(key, flags); // we are reading from the main DB - if (!value.IsNull() && _pruningContext?.DuplicateReads == true) + if (!value.IsNull() && _pruningContext?.DuplicateReads == true && (flags & ReadFlags.SkipDuplicateRead) == 0) { Duplicate(_pruningContext.CloningDb, key, value, WriteFlags.None); } @@ -126,6 +126,8 @@ public void Dispose() public IEnumerable> GetAll(bool ordered = false) => _currentDb.GetAll(ordered); + public IEnumerable GetAllKeys(bool ordered = false) => _currentDb.GetAllKeys(ordered); + public IEnumerable GetAllValues(bool ordered = false) => _currentDb.GetAllValues(ordered); // we need to remove from both DB's diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index d5d0cab97c2..1eb416712ca 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -12,6 +12,7 @@ public interface IDb : IKeyValueStoreWithBatching, IDbMeta, IDisposable string Name { get; } KeyValuePair[] this[byte[][] keys] { get; } IEnumerable> GetAll(bool ordered = false); + IEnumerable GetAllKeys(bool ordered = false); IEnumerable GetAllValues(bool ordered = false); public IReadOnlyDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyDb(this, createInMemWriteStore); diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index 7572adf7733..24a9c820255 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -88,6 +88,8 @@ public void Clear() public IEnumerable> GetAll(bool ordered = false) => _db; + public IEnumerable GetAllKeys(bool ordered = false) => Keys; + public IEnumerable GetAllValues(bool ordered = false) => Values; public virtual IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/NullDb.cs b/src/Nethermind/Nethermind.Db/NullDb.cs index 0d84f04b734..b914c537b5a 100644 --- a/src/Nethermind/Nethermind.Db/NullDb.cs +++ b/src/Nethermind/Nethermind.Db/NullDb.cs @@ -54,6 +54,7 @@ public void Clear() { } public IEnumerable> GetAll(bool ordered = false) => Enumerable.Empty>(); + public IEnumerable GetAllKeys(bool ordered = false) => Enumerable.Empty(); public IEnumerable GetAllValues(bool ordered = false) => Enumerable.Empty(); public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index 4806e91ff0f..26f1628b720 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -63,6 +63,8 @@ public KeyValuePair[] this[byte[][] keys] public IEnumerable> GetAll(bool ordered = false) => _memDb.GetAll(); + public IEnumerable GetAllKeys(bool ordered = false) => _memDb.GetAllKeys(); + public IEnumerable GetAllValues(bool ordered = false) => _memDb.GetAllValues(); public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs index 499f2104b2d..5aea9aefd6d 100644 --- a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs +++ b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs @@ -99,6 +99,8 @@ public void Clear() public IEnumerable> GetAll(bool ordered = false) => _cache; + public IEnumerable GetAllKeys(bool ordered = false) => _cache.Keys; + public IEnumerable GetAllValues(bool ordered = false) => _cache.Values; public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs b/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs index 2825bb7e5a8..1c3c0599562 100644 --- a/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs +++ b/src/Nethermind/Nethermind.Evm.Benchmark/EvmStackBenchmarks.cs @@ -47,27 +47,6 @@ public UInt256 Uint256(UInt256 v) return value; } - [Benchmark(OperationsPerInvoke = 4)] - [ArgumentsSource(nameof(ValueSource))] - public Int256.Int256 Int256(UInt256 v) - { - EvmStack stack = new(_stack.AsSpan(), 0, NullTxTracer.Instance); - - stack.PushSignedInt256(new Int256.Int256(v)); - stack.PopSignedInt256(out Int256.Int256 value); - - stack.PushSignedInt256(value); - stack.PopSignedInt256(out value); - - stack.PushSignedInt256(value); - stack.PopSignedInt256(out value); - - stack.PushSignedInt256(value); - stack.PopSignedInt256(out value); - - return value; - } - [Benchmark(OperationsPerInvoke = 4)] public byte Byte() { diff --git a/src/Nethermind/Nethermind.Evm/EvmStack.cs b/src/Nethermind/Nethermind.Evm/EvmStack.cs index 48dc9ae49a2..b59af410489 100644 --- a/src/Nethermind/Nethermind.Evm/EvmStack.cs +++ b/src/Nethermind/Nethermind.Evm/EvmStack.cs @@ -218,13 +218,6 @@ public void PopLimbo() } } - public void PopSignedInt256(out Int256.Int256 result) - { - // tail call into UInt256 - Unsafe.SkipInit(out result); - PopUInt256(out Unsafe.As(ref result)); - } - /// /// Pops an Uint256 written in big endian. /// @@ -233,9 +226,11 @@ public void PopSignedInt256(out Int256.Int256 result) /// All it does is and then reverse endianness if needed. Then it creates . /// /// The returned value. - public void PopUInt256(out UInt256 result) + public bool PopUInt256(out UInt256 result) { + Unsafe.SkipInit(out result); ref byte bytes = ref PopBytesByRef(); + if (Unsafe.IsNullRef(ref bytes)) return false; if (Avx2.IsSupported) { @@ -271,6 +266,8 @@ public void PopUInt256(out UInt256 result) result = new UInt256(u0, u1, u2, u3); } + + return true; } public readonly bool PeekUInt256IsZero() @@ -300,7 +297,7 @@ public Address PopAddress() { if (Head-- == 0) { - EvmStack.ThrowEvmStackUnderflowException(); + return null; } return new Address(_bytes.Slice(Head * WordSize + WordSize - AddressSize, AddressSize).ToArray()); @@ -310,7 +307,7 @@ public ref byte PopBytesByRef() { if (Head-- == 0) { - EvmStack.ThrowEvmStackUnderflowException(); + return ref Unsafe.NullRef(); } return ref _bytes[Head * WordSize]; @@ -356,9 +353,9 @@ public void PushLeftPaddedBytes(Span value, int paddingLength) } } - public void Dup(in int depth) + public bool Dup(in int depth) { - EnsureDepth(depth); + if (!EnsureDepth(depth)) return false; ref byte bytes = ref MemoryMarshal.GetReference(_bytes); @@ -376,19 +373,23 @@ public void Dup(in int depth) { EvmStack.ThrowEvmStackOverflowException(); } + + return true; } - public readonly void EnsureDepth(int depth) + public readonly bool EnsureDepth(int depth) { if (Head < depth) { - EvmStack.ThrowEvmStackUnderflowException(); + return false; } + + return true; } - public readonly void Swap(int depth) + public readonly bool Swap(int depth) { - EnsureDepth(depth); + if (!EnsureDepth(depth)) return false; ref byte bytes = ref MemoryMarshal.GetReference(_bytes); @@ -403,6 +404,8 @@ public readonly void Swap(int depth) { Trace(depth); } + + return true; } private readonly void Trace(int depth) diff --git a/src/Nethermind/Nethermind.Evm/Metrics.cs b/src/Nethermind/Nethermind.Evm/Metrics.cs index b904d0cd249..f9d16abd6c3 100644 --- a/src/Nethermind/Nethermind.Evm/Metrics.cs +++ b/src/Nethermind/Nethermind.Evm/Metrics.cs @@ -90,23 +90,23 @@ public class Metrics [Description("Number of contract create calls.")] public static long Creates { get; set; } internal static long Transactions { get; set; } - internal static decimal AveGasPrice { get; set; } - internal static decimal MinGasPrice { get; set; } = decimal.MaxValue; - internal static decimal MaxGasPrice { get; set; } - internal static decimal EstMedianGasPrice { get; set; } + internal static float AveGasPrice { get; set; } + internal static float MinGasPrice { get; set; } = float.MaxValue; + internal static float MaxGasPrice { get; set; } + internal static float EstMedianGasPrice { get; set; } internal static long BlockTransactions { get; set; } - internal static decimal BlockAveGasPrice { get; set; } - internal static decimal BlockMinGasPrice { get; set; } = decimal.MaxValue; - internal static decimal BlockMaxGasPrice { get; set; } - internal static decimal BlockEstMedianGasPrice { get; set; } + internal static float BlockAveGasPrice { get; set; } + internal static float BlockMinGasPrice { get; set; } = float.MaxValue; + internal static float BlockMaxGasPrice { get; set; } + internal static float BlockEstMedianGasPrice { get; set; } public static void ResetBlockStats() { BlockTransactions = 0; - BlockAveGasPrice = 0m; - BlockMaxGasPrice = 0m; - BlockEstMedianGasPrice = 0m; - BlockMinGasPrice = decimal.MaxValue; + BlockAveGasPrice = 0.0f; + BlockMaxGasPrice = 0.0f; + BlockEstMedianGasPrice = 0.0f; + BlockMinGasPrice = float.MaxValue; } } diff --git a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs index 5c456e673b8..c43469bd91c 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/BlockReceiptsTracer.cs @@ -12,7 +12,7 @@ namespace Nethermind.Evm.Tracing; public class BlockReceiptsTracer : IBlockTracer, ITxTracer, IJournal, ITxTracerWrapper { - private Block _block = null!; + protected Block Block = null!; public bool IsTracingReceipt => true; public bool IsTracingActions => _currentTxTracer.IsTracingActions; public bool IsTracingOpLevelStorage => _currentTxTracer.IsTracingOpLevelStorage; @@ -42,7 +42,8 @@ public void MarkAsSuccess(Address recipient, long gasSpent, byte[] output, LogEn if (_currentTxTracer.IsTracingReceipt) { - _currentTxTracer.MarkAsSuccess(recipient, gasSpent, output, logs); + // TODO: is no stateRoot a bug? + _currentTxTracer.MarkAsSuccess(recipient, gasSpent, output, logs, null); } } @@ -58,30 +59,31 @@ public void MarkAsFailed(Address recipient, long gasSpent, byte[] output, string if (_currentTxTracer.IsTracingReceipt) { - _currentTxTracer.MarkAsFailed(recipient, gasSpent, output, error); + // TODO: is no stateRoot a bug? + _currentTxTracer.MarkAsFailed(recipient, gasSpent, output, error, null); } } - private TxReceipt BuildFailedReceipt(Address recipient, long gasSpent, string error, Hash256? stateRoot = null) + protected TxReceipt BuildFailedReceipt(Address recipient, long gasSpent, string error, Hash256? stateRoot) { TxReceipt receipt = BuildReceipt(recipient, gasSpent, StatusCode.Failure, Array.Empty(), stateRoot); receipt.Error = error; return receipt; } - private TxReceipt BuildReceipt(Address recipient, long spentGas, byte statusCode, LogEntry[] logEntries, Hash256? stateRoot = null) + protected virtual TxReceipt BuildReceipt(Address recipient, long spentGas, byte statusCode, LogEntry[] logEntries, Hash256? stateRoot) { - Transaction transaction = _currentTx!; + Transaction transaction = CurrentTx!; TxReceipt txReceipt = new() { Logs = logEntries, TxType = transaction.Type, Bloom = logEntries.Length == 0 ? Bloom.Empty : new Bloom(logEntries), - GasUsedTotal = _block.GasUsed, + GasUsedTotal = Block.GasUsed, StatusCode = statusCode, Recipient = transaction.IsContractCreation ? null : recipient, - BlockHash = _block.Hash, - BlockNumber = _block.Number, + BlockHash = Block.Hash, + BlockNumber = Block.Number, Index = _currentIndex, GasUsed = spentGas, Sender = transaction.SenderAddress, @@ -193,7 +195,7 @@ public void ReportFees(UInt256 fees, UInt256 burntFees) private ITxTracer _currentTxTracer = NullTxTracer.Instance; private int _currentIndex; private readonly List _txReceipts = new(); - private Transaction? _currentTx; + protected Transaction? CurrentTx; public IReadOnlyList TxReceipts => _txReceipts; public TxReceipt LastReceipt => _txReceipts[^1]; public bool IsTracingRewards => _otherTracer.IsTracingRewards; @@ -211,7 +213,7 @@ public void Restore(int snapshot) _txReceipts.RemoveAt(_txReceipts.Count - 1); } - _block.Header.GasUsed = _txReceipts.Count > 0 ? _txReceipts.Last().GasUsedTotal : 0; + Block.Header.GasUsed = _txReceipts.Count > 0 ? _txReceipts.Last().GasUsedTotal : 0; } public void ReportReward(Address author, string rewardType, UInt256 rewardValue) => @@ -224,7 +226,7 @@ public void StartNewBlockTrace(Block block) throw new InvalidOperationException("other tracer not set in receipts tracer"); } - _block = block; + Block = block; _currentIndex = 0; _txReceipts.Clear(); @@ -233,7 +235,7 @@ public void StartNewBlockTrace(Block block) public ITxTracer StartNewTxTrace(Transaction? tx) { - _currentTx = tx; + CurrentTx = tx; _currentTxTracer = _otherTracer.StartNewTxTrace(tx); return _currentTxTracer; } @@ -250,7 +252,7 @@ public void EndBlockTrace() if (_txReceipts.Count > 0) { Bloom blockBloom = new(); - _block.Header.Bloom = blockBloom; + Block.Header.Bloom = blockBloom; for (int index = 0; index < _txReceipts.Count; index++) { TxReceipt? receipt = _txReceipts[index]; diff --git a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/Engine.cs b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/Engine.cs index 1b5e26d2ba3..4ff2da94a46 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/Engine.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/Engine.cs @@ -135,13 +135,11 @@ public Engine(IReleaseSpec spec) private ITypedArray Slice(object input, long start, long end) { ArgumentNullException.ThrowIfNull(input); + var bytes = input.ToBytes(); - if (start < 0 || end < start || end > Array.MaxLength) - { - throw new ArgumentOutOfRangeException(nameof(start), $"tracer accessed out of bound memory: offset {start}, end {end}"); - } - - return input.ToBytes().Slice((int)start, (int)(end - start)).ToTypedScriptArray(); + return start < 0 || end < start || end > bytes.Length + ? throw new ArgumentOutOfRangeException(nameof(start), $"tracer accessed out of bound memory: available {bytes.Length}, offset {start}, size {end - start}") + : bytes.Slice((int)start, (int)(end - start)).ToTypedScriptArray(); } /// diff --git a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/JavaScriptConverter.cs b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/JavaScriptConverter.cs index 2523f4764c6..9d455e5098a 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/JavaScriptConverter.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/GethStyle/JavaScript/JavaScriptConverter.cs @@ -35,7 +35,7 @@ public static class JavaScriptConverter public static Address ToAddress(this object address) => address switch { - string hexString => Address.TryParseVariableLength(hexString, out Address parsedAddress) + string hexString => Address.TryParseVariableLength(hexString, out Address parsedAddress, true) ? parsedAddress : throw new ArgumentException("Not correct address", nameof(address)), _ => new Address(address.ToBytes()) diff --git a/src/Nethermind/Nethermind.Evm/Tracing/TraceMemory.cs b/src/Nethermind/Nethermind.Evm/Tracing/TraceMemory.cs index cec5cf8726e..cc63b903019 100644 --- a/src/Nethermind/Nethermind.Evm/Tracing/TraceMemory.cs +++ b/src/Nethermind/Nethermind.Evm/Tracing/TraceMemory.cs @@ -21,7 +21,7 @@ public TraceMemory(ulong size, ReadOnlyMemory memory) public string[] ToHexWordList() { - string[] memory = new string[((int)Size / EvmPooledMemory.WordSize) + ((Size % EvmPooledMemory.WordSize == 0) ? 0 : 1)]; + string[] memory = new string[(int)Size / EvmPooledMemory.WordSize + (Size % EvmPooledMemory.WordSize == 0 ? 0 : 1)]; int traceLocation = 0; int i = 0; @@ -45,21 +45,23 @@ public string[] ToHexWordList() return memory; } + private const int MemoryPadLimit = 1024 * 1024; public ReadOnlySpan Slice(int start, int length) { - if ((ulong)start + (ulong)length > Size) - { - throw new IndexOutOfRangeException("Requested memory range is out of bounds."); - } + ArgumentOutOfRangeException.ThrowIfNegative(start, nameof(start)); + ArgumentOutOfRangeException.ThrowIfNegative(length, nameof(length)); ReadOnlySpan span = _memory.Span; - if (start + length > _memory.Length) + if (start + length > span.Length) { + int paddingNeeded = start + length - span.Length; + if (paddingNeeded > MemoryPadLimit) throw new InvalidOperationException($"reached limit for padding memory slice: {paddingNeeded}"); byte[] result = new byte[length]; - for (int i = 0, index = start; index < _memory.Length; i++, index++) + int overlap = span.Length - start; + if (overlap > 0) { - result[i] = span[index]; + span.Slice(start, overlap).CopyTo(result.AsSpan(0, overlap)); } return result; @@ -68,13 +70,8 @@ public ReadOnlySpan Slice(int start, int length) return span.Slice(start, length); } - public BigInteger GetUint(int offset) - { - if (offset < 0 || (ulong)(offset + EvmPooledMemory.WordSize) > Size) - { - throw new ArgumentOutOfRangeException(nameof(offset), $"tracer accessed out of bound memory: available {Size}, offset {offset}, size {EvmPooledMemory.WordSize}"); - } - - return new BigInteger(Slice(offset, EvmPooledMemory.WordSize), true, true); - } + public BigInteger GetUint(int offset) => + offset < 0 || (ulong)(offset + EvmPooledMemory.WordSize) > Size + ? throw new ArgumentOutOfRangeException(nameof(offset), $"tracer accessed out of bound memory: available {Size}, offset {offset}, size {EvmPooledMemory.WordSize}") + : new BigInteger(Slice(offset, EvmPooledMemory.WordSize), true, true); } diff --git a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs index 8eb05afb4be..73fcfc54c25 100644 --- a/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Evm/TransactionProcessing/TransactionProcessor.cs @@ -117,7 +117,7 @@ protected virtual void Execute(Transaction tx, BlockExecutionContext blCtx, ITxT if (opts == ExecutionOptions.Commit || opts == ExecutionOptions.None) { - decimal gasPrice = (decimal)effectiveGasPrice / 1_000_000_000m; + float gasPrice = (float)((double)effectiveGasPrice / 1_000_000_000.0); Metrics.MinGasPrice = Math.Min(gasPrice, Metrics.MinGasPrice); Metrics.MaxGasPrice = Math.Max(gasPrice, Metrics.MaxGasPrice); @@ -125,11 +125,11 @@ protected virtual void Execute(Transaction tx, BlockExecutionContext blCtx, ITxT Metrics.BlockMaxGasPrice = Math.Max(gasPrice, Metrics.BlockMaxGasPrice); Metrics.AveGasPrice = (Metrics.AveGasPrice * Metrics.Transactions + gasPrice) / (Metrics.Transactions + 1); - Metrics.EstMedianGasPrice += Metrics.AveGasPrice * 0.01m * decimal.Sign(gasPrice - Metrics.EstMedianGasPrice); + Metrics.EstMedianGasPrice += Metrics.AveGasPrice * 0.01f * float.Sign(gasPrice - Metrics.EstMedianGasPrice); Metrics.Transactions++; Metrics.BlockAveGasPrice = (Metrics.BlockAveGasPrice * Metrics.BlockTransactions + gasPrice) / (Metrics.BlockTransactions + 1); - Metrics.BlockEstMedianGasPrice += Metrics.BlockAveGasPrice * 0.01m * decimal.Sign(gasPrice - Metrics.BlockEstMedianGasPrice); + Metrics.BlockEstMedianGasPrice += Metrics.BlockAveGasPrice * 0.01f * float.Sign(gasPrice - Metrics.BlockEstMedianGasPrice); Metrics.BlockTransactions++; } diff --git a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs index c168656d7ba..3ddd12fbc2d 100644 --- a/src/Nethermind/Nethermind.Evm/VirtualMachine.cs +++ b/src/Nethermind/Nethermind.Evm/VirtualMachine.cs @@ -811,8 +811,8 @@ private CallResult ExecuteCode= BigInt32) { - stack.EnsureDepth(1); + if (!stack.EnsureDepth(1)) goto StackUnderflow; break; } @@ -1018,8 +1018,8 @@ private CallResult ExecuteCode b) { stack.PushOne(); @@ -1052,8 +1052,8 @@ private CallResult ExecuteCode(ref a).CompareTo(As(ref b)) < 0) { @@ -1070,8 +1070,8 @@ private CallResult ExecuteCode(ref a).CompareTo(As(ref b)) > 0) { stack.PushOne(); @@ -1087,8 +1087,8 @@ private CallResult ExecuteCode aVec = ReadUnaligned>(ref stack.PopBytesByRef()); - Vector256 bVec = ReadUnaligned>(ref stack.PopBytesByRef()); + ref byte bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + Vector256 aVec = ReadUnaligned>(ref bytesRef); + + bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + Vector256 bVec = ReadUnaligned>(ref bytesRef); WriteUnaligned(ref stack.PushBytesRef(), Vector256.BitwiseAnd(aVec, bVec)); break; @@ -1130,8 +1135,13 @@ private CallResult ExecuteCode aVec = ReadUnaligned>(ref stack.PopBytesByRef()); - Vector256 bVec = ReadUnaligned>(ref stack.PopBytesByRef()); + ref byte bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + Vector256 aVec = ReadUnaligned>(ref bytesRef); + + bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + Vector256 bVec = ReadUnaligned>(ref bytesRef); WriteUnaligned(ref stack.PushBytesRef(), Vector256.BitwiseOr(aVec, bVec)); break; @@ -1140,8 +1150,13 @@ private CallResult ExecuteCode aVec = ReadUnaligned>(ref stack.PopBytesByRef()); - Vector256 bVec = ReadUnaligned>(ref stack.PopBytesByRef()); + ref byte bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + Vector256 aVec = ReadUnaligned>(ref bytesRef); + + bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + Vector256 bVec = ReadUnaligned>(ref bytesRef); WriteUnaligned(ref stack.PushBytesRef(), Vector256.Xor(aVec, bVec)); break; @@ -1150,7 +1165,10 @@ private CallResult ExecuteCode negVec = Vector256.OnesComplement(ReadUnaligned>(ref stack.PopBytesByRef())); + ref byte bytesRef = ref stack.PopBytesByRef(); + if (IsNullRef(ref bytesRef)) goto StackUnderflow; + + Vector256 negVec = Vector256.OnesComplement(ReadUnaligned>(ref bytesRef)); WriteUnaligned(ref stack.PushBytesRef(), negVec); break; @@ -1159,7 +1177,7 @@ private CallResult ExecuteCode= BigInt32) @@ -1182,8 +1200,8 @@ private CallResult ExecuteCode _returnDataBuffer.Length) @@ -1432,7 +1455,7 @@ private CallResult ExecuteCode long.MaxValue ? long.MaxValue : (long)a; Hash256 blockHash = _blockhashProvider.GetBlockhash(blkCtx.Header, number); stack.PushBytes(blockHash != null ? blockHash.Bytes : BytesZero32); @@ -1528,7 +1551,7 @@ private CallResult ExecuteCode(vmState, ref stack, ref gasAvailable, spec)) - goto OutOfGas; + exceptionType = InstructionSStore(vmState, ref stack, ref gasAvailable, spec); + if (exceptionType != EvmExceptionType.None) goto ReturnFailure; break; } @@ -1633,7 +1656,7 @@ private CallResult ExecuteCode(vmState, ref stack, ref gasAvailable, spec, instruction, out returnData); - if (exceptionType != EvmExceptionType.None) - { - goto ReturnFailure; - } + if (exceptionType != EvmExceptionType.None) goto ReturnFailure; + if (returnData is null) { break; @@ -1844,7 +1867,8 @@ private CallResult ExecuteCode= 256UL) { stack.PopLimbo(); @@ -1882,7 +1907,7 @@ private CallResult ExecuteCode= 256) { stack.PopLimbo(); @@ -1903,7 +1928,7 @@ private CallResult ExecuteCode> (int)a.u0; stack.PushUInt256(in result); } @@ -1916,8 +1941,8 @@ private CallResult ExecuteCode= BigInt256) { if (As(ref b).Sign >= 0) @@ -1944,6 +1969,7 @@ private CallResult ExecuteCode( if (instruction == Instruction.DELEGATECALL && !spec.DelegateCallEnabled || instruction == Instruction.STATICCALL && !spec.StaticCallEnabled) return EvmExceptionType.BadInstruction; - stack.PopUInt256(out UInt256 gasLimit); + if (!stack.PopUInt256(out UInt256 gasLimit)) return EvmExceptionType.StackUnderflow; Address codeSource = stack.PopAddress(); + if (codeSource is null) return EvmExceptionType.StackUnderflow; if (!ChargeAccountAccessGas(ref gasAvailable, vmState, codeSource, spec)) return EvmExceptionType.OutOfGas; @@ -2175,15 +2205,15 @@ private EvmExceptionType InstructionCall( callValue = env.Value; break; default: - stack.PopUInt256(out callValue); + if (!stack.PopUInt256(out callValue)) return EvmExceptionType.StackUnderflow; break; } UInt256 transferValue = instruction == Instruction.DELEGATECALL ? UInt256.Zero : callValue; - stack.PopUInt256(out UInt256 dataOffset); - stack.PopUInt256(out UInt256 dataLength); - stack.PopUInt256(out UInt256 outputOffset); - stack.PopUInt256(out UInt256 outputLength); + if (!stack.PopUInt256(out UInt256 dataOffset)) return EvmExceptionType.StackUnderflow; + if (!stack.PopUInt256(out UInt256 dataLength)) return EvmExceptionType.StackUnderflow; + if (!stack.PopUInt256(out UInt256 outputOffset)) return EvmExceptionType.StackUnderflow; + if (!stack.PopUInt256(out UInt256 outputLength)) return EvmExceptionType.StackUnderflow; if (vmState.IsStatic && !transferValue.IsZero && instruction != Instruction.CALLCODE) return EvmExceptionType.StaticCallViolation; @@ -2303,48 +2333,53 @@ private EvmExceptionType InstructionCall( } [SkipLocalsInit] - private static bool InstructionRevert(EvmState vmState, ref EvmStack stack, ref long gasAvailable, out object returnData) + private static EvmExceptionType InstructionRevert(EvmState vmState, ref EvmStack stack, ref long gasAvailable, out object returnData) where TTracing : struct, IIsTracing { - stack.PopUInt256(out UInt256 position); - stack.PopUInt256(out UInt256 length); + SkipInit(out returnData); + + if (!stack.PopUInt256(out UInt256 position) || + !stack.PopUInt256(out UInt256 length)) + return EvmExceptionType.StackUnderflow; if (!UpdateMemoryCost(vmState, ref gasAvailable, in position, in length)) { - returnData = null; - return false; + return EvmExceptionType.OutOfGas; } returnData = vmState.Memory.Load(in position, in length).ToArray(); - return true; + return EvmExceptionType.None; } [SkipLocalsInit] - private static bool InstructionReturn(EvmState vmState, ref EvmStack stack, ref long gasAvailable, out object returnData) + private static EvmExceptionType InstructionReturn(EvmState vmState, ref EvmStack stack, ref long gasAvailable, out object returnData) where TTracing : struct, IIsTracing { - stack.PopUInt256(out UInt256 position); - stack.PopUInt256(out UInt256 length); + SkipInit(out returnData); + + if (!stack.PopUInt256(out UInt256 position) || + !stack.PopUInt256(out UInt256 length)) + return EvmExceptionType.StackUnderflow; if (!UpdateMemoryCost(vmState, ref gasAvailable, in position, in length)) { - returnData = null; - return false; + return EvmExceptionType.OutOfGas; } returnData = vmState.Memory.Load(in position, in length).ToArray(); - return true; + return EvmExceptionType.None; } [SkipLocalsInit] - private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack stack, ref long gasAvailable, IReleaseSpec spec) + private EvmExceptionType InstructionSelfDestruct(EvmState vmState, ref EvmStack stack, ref long gasAvailable, IReleaseSpec spec) where TTracing : struct, IIsTracing { Metrics.SelfDestructs++; Address inheritor = stack.PopAddress(); - if (!ChargeAccountAccessGas(ref gasAvailable, vmState, inheritor, spec, false)) return false; + if (inheritor is null) return EvmExceptionType.StackUnderflow; + if (!ChargeAccountAccessGas(ref gasAvailable, vmState, inheritor, spec, false)) return EvmExceptionType.OutOfGas; Address executingAccount = vmState.Env.ExecutingAccount; bool createInSameTx = vmState.CreateList.Contains(executingAccount); @@ -2355,13 +2390,13 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack(EvmState vmState, ref EvmStack(EvmState vmState, ref EvmStack stack, ref long gasAvailable, IReleaseSpec spec, Instruction instruction) + private (EvmExceptionType exceptionType, EvmState? callState) InstructionCreate(EvmState vmState, ref EvmStack stack, ref long gasAvailable, IReleaseSpec spec, Instruction instruction) where TTracing : struct, IIsTracing { ref readonly ExecutionEnvironment env = ref vmState.Env; @@ -2392,9 +2427,11 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack salt = default; if (instruction == Instruction.CREATE2) { @@ -2404,7 +2441,7 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack spec.MaxInitCodeSize) return (outOfGas: true, null); + if (initCodeLength > spec.MaxInitCodeSize) return (EvmExceptionType.OutOfGas, null); } long gasCost = GasCostOf.Create + @@ -2413,9 +2450,9 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack= MaxCallDepth) // TODO: fragile ordering / potential vulnerability for different clients @@ -2423,7 +2460,7 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack(); stack.PushZero(); - return (outOfGas: false, null); + return (EvmExceptionType.None, null); } Span initCode = vmState.Memory.LoadSpan(in memoryPositionOfInitCode, initCodeLength); @@ -2433,7 +2470,7 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack(); stack.PushZero(); - return (outOfGas: false, null); + return (EvmExceptionType.None, null); } UInt256 accountNonce = _state.GetNonce(env.ExecutingAccount); @@ -2442,14 +2479,14 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack(); stack.PushZero(); - return (outOfGas: false, null); + return (EvmExceptionType.None, null); } if (typeof(TTracing) == typeof(IsTracing)) EndInstructionTrace(gasAvailable, vmState.Memory?.Size ?? 0); // todo: === below is a new call - refactor / move long callGas = spec.Use63Over64Rule ? gasAvailable - gasAvailable / 64L : gasAvailable; - if (!UpdateGas(callGas, ref gasAvailable)) return (outOfGas: true, null); + if (!UpdateGas(callGas, ref gasAvailable)) return (EvmExceptionType.OutOfGas, null); Address contractAddress = instruction == Instruction.CREATE ? ContractAddress.From(env.ExecutingAccount, _state.GetNonce(env.ExecutingAccount)) @@ -2473,7 +2510,7 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack(); stack.PushZero(); - return (outOfGas: false, null); + return (EvmExceptionType.None, null); } if (accountExists) @@ -2521,20 +2558,20 @@ private bool InstructionSelfDestruct(EvmState vmState, ref EvmStack(EvmState vmState, ref EvmStack stack, ref long gasAvailable, Instruction instruction) + private static EvmExceptionType InstructionLog(EvmState vmState, ref EvmStack stack, ref long gasAvailable, Instruction instruction) where TTracing : struct, IIsTracing { - stack.PopUInt256(out UInt256 position); - stack.PopUInt256(out UInt256 length); + if (!stack.PopUInt256(out UInt256 position)) return EvmExceptionType.StackUnderflow; + if (!stack.PopUInt256(out UInt256 length)) return EvmExceptionType.StackUnderflow; long topicsCount = instruction - Instruction.LOG0; - if (!UpdateMemoryCost(vmState, ref gasAvailable, in position, length)) return false; + if (!UpdateMemoryCost(vmState, ref gasAvailable, in position, length)) return EvmExceptionType.OutOfGas; if (!UpdateGas( GasCostOf.Log + topicsCount * GasCostOf.LogTopic + - (long)length * GasCostOf.LogData, ref gasAvailable)) return false; + (long)length * GasCostOf.LogData, ref gasAvailable)) return EvmExceptionType.OutOfGas; ReadOnlyMemory data = vmState.Memory.Load(in position, length); Hash256[] topics = new Hash256[topicsCount]; @@ -2549,26 +2586,26 @@ private static bool InstructionLog(EvmState vmState, ref EvmStack(EvmState vmState, ref EvmStack stack, ref long gasAvailable, IReleaseSpec spec) + private EvmExceptionType InstructionSStore(EvmState vmState, ref EvmStack stack, ref long gasAvailable, IReleaseSpec spec) where TTracingInstructions : struct, IIsTracing where TTracingRefunds : struct, IIsTracing where TTracingStorage : struct, IIsTracing { // fail fast before the first storage read if gas is not enough even for reset - if (!spec.UseNetGasMetering && !UpdateGas(spec.GetSStoreResetCost(), ref gasAvailable)) return false; + if (!spec.UseNetGasMetering && !UpdateGas(spec.GetSStoreResetCost(), ref gasAvailable)) return EvmExceptionType.OutOfGas; if (spec.UseNetGasMeteringWithAStipendFix) { if (typeof(TTracingRefunds) == typeof(IsTracing)) _txTracer.ReportExtraGasPressure(GasCostOf.CallStipend - spec.GetNetMeteredSStoreCost() + 1); - if (gasAvailable <= GasCostOf.CallStipend) return false; + if (gasAvailable <= GasCostOf.CallStipend) return EvmExceptionType.OutOfGas; } - stack.PopUInt256(out UInt256 result); + if (!stack.PopUInt256(out UInt256 result)) return EvmExceptionType.StackUnderflow; Span bytes = stack.PopWord256(); bool newIsZero = bytes.IsZero(); if (!newIsZero) @@ -2587,7 +2624,7 @@ private bool InstructionSStore currentValue = _state.Get(in storageCell); // Console.WriteLine($"current: {currentValue.ToHexString()} newValue {newValue.ToHexString()}"); @@ -2608,14 +2645,14 @@ private bool InstructionSStore(long gasAvailable, EvmExceptionType exceptionType) @@ -2714,6 +2751,7 @@ private CallResult GetFailureReturn(long gasAvailable, Evm EvmExceptionType.InvalidSubroutineEntry => CallResult.InvalidSubroutineEntry, EvmExceptionType.InvalidSubroutineReturn => CallResult.InvalidSubroutineReturn, EvmExceptionType.StackOverflow => CallResult.StackOverflowException, + EvmExceptionType.StackUnderflow => CallResult.StackUnderflowException, EvmExceptionType.InvalidJumpDestination => CallResult.InvalidJumpDestination, EvmExceptionType.AccessViolation => CallResult.AccessViolationException, _ => throw new ArgumentOutOfRangeException(nameof(exceptionType), exceptionType, "") diff --git a/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs b/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs index 12346222d54..5a40fd961f1 100644 --- a/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs +++ b/src/Nethermind/Nethermind.Facade/BlockchainBridge.cs @@ -103,7 +103,7 @@ public Block? HeadBlock return (null, null, 0); } - public (TxReceipt? Receipt, Transaction Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash) + public (TxReceipt? Receipt, Transaction Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash, bool checkTxnPool = true) { Hash256 blockHash = _receiptFinder.FindBlockHash(txHash); if (blockHash is not null) @@ -113,7 +113,7 @@ public Block? HeadBlock return (txReceipt, block?.Transactions[txReceipt.Index], block?.BaseFeePerGas); } - if (_txPool.TryGetPendingTransaction(txHash, out Transaction? transaction)) + if (checkTxnPool && _txPool.TryGetPendingTransaction(txHash, out Transaction? transaction)) { return (null, transaction, null); } @@ -302,10 +302,28 @@ public IEnumerable GetLogs( IEnumerable? topics = null, CancellationToken cancellationToken = default) { - LogFilter filter = _filterStore.CreateLogFilter(fromBlock, toBlock, address, topics, false); + LogFilter filter = GetFilter(fromBlock, toBlock, address, topics); return _logFinder.FindLogs(filter, cancellationToken); } + public LogFilter GetFilter( + BlockParameter fromBlock, + BlockParameter toBlock, + object? address = null, + IEnumerable? topics = null) + { + return _filterStore.CreateLogFilter(fromBlock, toBlock, address, topics, false); + } + + public IEnumerable GetLogs( + LogFilter filter, + BlockHeader fromBlock, + BlockHeader toBlock, + CancellationToken cancellationToken = default) + { + return _logFinder.FindLogs(filter, fromBlock, toBlock, cancellationToken); + } + public bool TryGetLogs(int filterId, out IEnumerable filterLogs, CancellationToken cancellationToken = default) { LogFilter? filter; @@ -379,6 +397,11 @@ public bool HasStateForRoot(Hash256 stateRoot) return _processingEnv.StateReader.HasStateForRoot(stateRoot); } + public IEnumerable FindLogs(LogFilter filter, BlockHeader fromBlock, BlockHeader toBlock, CancellationToken cancellationToken = default) + { + return _logFinder.FindLogs(filter, fromBlock, toBlock, cancellationToken); + } + public IEnumerable FindLogs(LogFilter filter, CancellationToken cancellationToken = default) { return _logFinder.FindLogs(filter, cancellationToken); diff --git a/src/Nethermind/Nethermind.Facade/Filters/ILogFinder.cs b/src/Nethermind/Nethermind.Facade/Filters/ILogFinder.cs index cc81d81b3ef..a9a2455c78b 100644 --- a/src/Nethermind/Nethermind.Facade/Filters/ILogFinder.cs +++ b/src/Nethermind/Nethermind.Facade/Filters/ILogFinder.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Threading; using Nethermind.Blockchain.Filters; +using Nethermind.Core; using Nethermind.Facade.Filters; namespace Nethermind.Blockchain.Find @@ -11,5 +12,6 @@ namespace Nethermind.Blockchain.Find public interface ILogFinder { IEnumerable FindLogs(LogFilter filter, CancellationToken cancellationToken = default); + IEnumerable FindLogs(LogFilter filter, BlockHeader fromBlock, BlockHeader toBlock, CancellationToken cancellationToken = default); } } diff --git a/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs b/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs index 8ef6790b9a4..a2a807d78ed 100644 --- a/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs +++ b/src/Nethermind/Nethermind.Facade/Filters/LogFinder.cs @@ -54,9 +54,18 @@ BlockHeader FindHeader(BlockParameter blockParameter, string name, bool headLimi _blockFinder.FindHeader(blockParameter, headLimit) ?? throw new ResourceNotFoundException($"Block not found: {name} {blockParameter}"); cancellationToken.ThrowIfCancellationRequested(); - var toBlock = FindHeader(filter.ToBlock, nameof(filter.ToBlock), false); + BlockHeader toBlock = FindHeader(filter.ToBlock, nameof(filter.ToBlock), false); + cancellationToken.ThrowIfCancellationRequested(); + BlockHeader fromBlock = filter.ToBlock == filter.FromBlock ? + toBlock : + FindHeader(filter.FromBlock, nameof(filter.FromBlock), false); + + return FindLogs(filter, fromBlock, toBlock, cancellationToken); + } + + public IEnumerable FindLogs(LogFilter filter, BlockHeader fromBlock, BlockHeader toBlock, CancellationToken cancellationToken = default) + { cancellationToken.ThrowIfCancellationRequested(); - var fromBlock = FindHeader(filter.FromBlock, nameof(filter.FromBlock), false); if (fromBlock.Number > toBlock.Number && toBlock.Number != 0) { diff --git a/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs b/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs index f08d4dff8ab..fdcd1ab1d3c 100644 --- a/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs +++ b/src/Nethermind/Nethermind.Facade/IBlockchainBridge.cs @@ -23,7 +23,7 @@ public interface IBlockchainBridge : ILogFinder Address? RecoverTxSender(Transaction tx); TxReceipt GetReceipt(Hash256 txHash); (TxReceipt? Receipt, TxGasInfo? GasInfo, int LogIndexStart) GetReceiptAndGasInfo(Hash256 txHash); - (TxReceipt? Receipt, Transaction Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash); + (TxReceipt? Receipt, Transaction Transaction, UInt256? baseFee) GetTransaction(Hash256 txHash, bool checkTxnPool = true); BlockchainBridge.CallOutput Call(BlockHeader header, Transaction tx, CancellationToken cancellationToken); BlockchainBridge.CallOutput EstimateGas(BlockHeader header, Transaction tx, CancellationToken cancellationToken); BlockchainBridge.CallOutput CreateAccessList(BlockHeader header, Transaction tx, CancellationToken cancellationToken, bool optimize); @@ -41,7 +41,10 @@ public interface IBlockchainBridge : ILogFinder FilterType GetFilterType(int filterId); FilterLog[] GetFilterLogs(int filterId); + LogFilter GetFilter(BlockParameter fromBlock, BlockParameter toBlock, object? address = null, IEnumerable? topics = null); + IEnumerable GetLogs(LogFilter filter, BlockHeader fromBlock, BlockHeader toBlock, CancellationToken cancellationToken = default); IEnumerable GetLogs(BlockParameter fromBlock, BlockParameter toBlock, object? address = null, IEnumerable? topics = null, CancellationToken cancellationToken = default); + bool TryGetLogs(int filterId, out IEnumerable filterLogs, CancellationToken cancellationToken = default); void RunTreeVisitor(ITreeVisitor treeVisitor, Hash256 stateRoot); bool HasStateForRoot(Hash256 stateRoot); diff --git a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs index 1a970d399ba..ac870740367 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs @@ -46,7 +46,7 @@ public virtual async Task Execute(CancellationToken _) try { bool useReceiptsDb = initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync; - bool useBlobsDb = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true }; + bool useBlobsDb = txPoolConfig.BlobsSupport.IsPersistentStorage(); InitDbApi(initConfig, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync); StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem); await dbInitializer.InitStandardDbsAsync(useReceiptsDb, useBlobsDb); diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs index cb86dcc64b5..bd4036393c9 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockTree.cs @@ -102,6 +102,11 @@ public Task Execute(CancellationToken cancellationToken) _set.LogFinder = logFinder; + if (initConfig.ExitOnBlockNumber != null) + { + new ExitOnBlockNumberHandler(blockTree, _get.ProcessExit!, initConfig.ExitOnBlockNumber.Value, _get.LogManager); + } + return Task.CompletedTask; } } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index 798b19e6b91..f87e07b38f7 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -73,7 +73,7 @@ protected virtual Task InitBlockchain() IStateReader stateReader = setApi.StateReader!; ITxPool txPool = _api.TxPool = CreateTxPool(); - ReceiptCanonicalityMonitor receiptCanonicalityMonitor = new(getApi.BlockTree, getApi.ReceiptStorage, _api.LogManager); + ReceiptCanonicalityMonitor receiptCanonicalityMonitor = new(getApi.ReceiptStorage, _api.LogManager); getApi.DisposeStack.Push(receiptCanonicalityMonitor); _api.ReceiptMonitor = receiptCanonicalityMonitor; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs index dabb9c414e0..ed059c86e65 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/DebugModuleTests.cs @@ -20,6 +20,7 @@ using Nethermind.Logging; using Nethermind.Serialization.Rlp; using NSubstitute; +using NSubstitute.ReturnsExtensions; using NUnit.Framework; namespace Nethermind.JsonRpc.Test.Modules; @@ -88,7 +89,7 @@ public async Task Get_block_rlp_by_hash() { BlockDecoder decoder = new(); Rlp rlp = decoder.Encode(Build.A.Block.WithNumber(1).TestObject); - debugBridge.GetBlockRlp(Keccak.Zero).Returns(rlp.Bytes); + debugBridge.GetBlockRlp(new BlockParameter(Keccak.Zero)).Returns(rlp.Bytes); DebugRpcModule rpcModule = new(LimboLogs.Instance, debugBridge, jsonRpcConfig); JsonRpcSuccessResponse? response = await RpcTest.TestRequest(rpcModule, "debug_getBlockRlpByHash", $"{Keccak.Zero.Bytes.ToHexString()}") as JsonRpcSuccessResponse; @@ -101,7 +102,7 @@ public async Task Get_block_rlp() BlockDecoder decoder = new(); IDebugBridge debugBridge = Substitute.For(); Rlp rlp = decoder.Encode(Build.A.Block.WithNumber(1).TestObject); - debugBridge.GetBlockRlp(1).Returns(rlp.Bytes); + debugBridge.GetBlockRlp(new BlockParameter(1)).Returns(rlp.Bytes); DebugRpcModule rpcModule = new(LimboLogs.Instance, debugBridge, jsonRpcConfig); JsonRpcSuccessResponse? response = await RpcTest.TestRequest(rpcModule, "debug_getBlockRlp", "1") as JsonRpcSuccessResponse; @@ -112,7 +113,7 @@ public async Task Get_block_rlp() [Test] public async Task Get_block_rlp_when_missing() { - debugBridge.GetBlockRlp(1).Returns((byte[])null!); + debugBridge.GetBlockRlp(new BlockParameter(1)).ReturnsNull(); DebugRpcModule rpcModule = new(LimboLogs.Instance, debugBridge, jsonRpcConfig); JsonRpcErrorResponse? response = await RpcTest.TestRequest(rpcModule, "debug_getBlockRlp", "1") as JsonRpcErrorResponse; @@ -125,7 +126,7 @@ public async Task Get_block_rlp_by_hash_when_missing() { BlockDecoder decoder = new(); Rlp rlp = decoder.Encode(Build.A.Block.WithNumber(1).TestObject); - debugBridge.GetBlockRlp(Keccak.Zero).Returns((byte[])null!); + debugBridge.GetBlockRlp(new BlockParameter(Keccak.Zero)).ReturnsNull(); DebugRpcModule rpcModule = new(LimboLogs.Instance, debugBridge, jsonRpcConfig); JsonRpcErrorResponse? response = await RpcTest.TestRequest(rpcModule, "debug_getBlockRlpByHash", $"{Keccak.Zero.Bytes.ToHexString()}") as JsonRpcErrorResponse; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs index 1505a222793..ddf1ed2da5b 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using FluentAssertions; using Nethermind.Blockchain; +using Nethermind.Blockchain.Filters; using Nethermind.Blockchain.Find; using Nethermind.Blockchain.Receipts; using Nethermind.Core; @@ -523,7 +524,7 @@ public async Task Eth_get_logs(string parameter, string expected) { using Context ctx = await Context.Create(); IBlockchainBridge bridge = Substitute.For(); - bridge.GetLogs(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any()) + bridge.GetLogs(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(new[] { new FilterLog(1, 0, 1, TestItem.KeccakA, 1, TestItem.KeccakB, TestItem.AddressA, new byte[] { 1, 2, 3 }, new[] { TestItem.KeccakC, TestItem.KeccakD }) }); bridge.FilterExists(1).Returns(true); @@ -538,10 +539,10 @@ public async Task Eth_get_logs_cancellation() { using Context ctx = await Context.Create(); IBlockchainBridge bridge = Substitute.For(); - bridge.GetLogs(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any()) + bridge.GetLogs(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Returns(c => { - return GetLogs(c.ArgAt(4)); + return GetLogs(c.ArgAt(3)); [DoesNotReturn] IEnumerable GetLogs(CancellationToken ct) @@ -564,7 +565,7 @@ public async Task Eth_get_logs_with_resourceNotFound(string parameter, string ex { using Context ctx = await Context.Create(); IBlockchainBridge bridge = Substitute.For(); - bridge.GetLogs(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any>(), Arg.Any()) + bridge.GetLogs(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) .Throws(new ResourceNotFoundException("resource not found message")); bridge.FilterExists(1).Returns(true); diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs index 123497a5aa1..e2c033e4925 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/SubscribeModuleTests.cs @@ -64,7 +64,7 @@ public void Setup() _filterStore = new FilterStore(); _jsonRpcDuplexClient = Substitute.For(); _jsonSerializer = new EthereumJsonSerializer(); - _receiptCanonicalityMonitor = new ReceiptCanonicalityMonitor(_blockTree, _receiptStorage, _logManager); + _receiptCanonicalityMonitor = new ReceiptCanonicalityMonitor(_receiptStorage, _logManager); _syncConfig = new SyncConfig(); IJsonSerializer jsonSerializer = new EthereumJsonSerializer(); @@ -114,6 +114,7 @@ private JsonRpcResult GetBlockAddedToMainResult(BlockReplacementEventArgs blockR })); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockReplacementEventArgs); + _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockReplacementEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(1000)).Should().Be(shouldReceiveResult); subscriptionId = newHeadSubscription.Id; @@ -133,6 +134,7 @@ private List GetLogsSubscriptionResult(Filter filter, BlockReplac })); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); + _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); semaphoreSlim.Wait(TimeSpan.FromMilliseconds(100)); subscriptionId = logsSubscription.Id; @@ -713,6 +715,7 @@ public void LogsSubscription_should_not_send_logs_of_new_txs_on_ReceiptsInserted BlockReplacementEventArgs blockEventArgs = new(block); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); + _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(200)); jsonRpcResults.Count.Should().Be(1); @@ -892,6 +895,7 @@ public async Task MultipleSubscriptions_concurrent_fast_messages(int messages) { BlockReplacementEventArgs eventArgs = new(Build.A.Block.TestObject); blockTree.BlockAddedToMain += Raise.EventWith(eventArgs); + _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), eventArgs); } }); @@ -1148,6 +1152,7 @@ public void LogsSubscription_can_send_logs_with_removed_txs_when_inserted() })); _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockEventArgs); + _receiptStorage.ReceiptsInserted += Raise.EventWith(new object(), blockEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(2000)); jsonRpcResults.Count.Should().Be(1); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs index eb45fb78b4c..c55f8e32c61 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugBridge.cs @@ -118,6 +118,31 @@ public void InsertReceipts(BlockParameter blockParameter, TxReceipt[] txReceipts _receiptStorage.Insert(block, txReceipts); } + public TxReceipt[]? GetReceiptsForBlock(BlockParameter blockParam) + { + SearchResult searchResult = _blockTree.SearchForBlock(blockParam); + if (searchResult.IsError) + { + throw new InvalidDataException(searchResult.Error); + } + + Block block = searchResult.Object; + return _receiptStorage.Get(block); + } + + public Transaction? GetTransactionFromHash(Hash256 txHash) + { + Hash256 blockHash = _receiptStorage.FindBlockHash(txHash); + SearchResult searchResult = _blockTree.SearchForBlock(new BlockParameter(blockHash)); + if (searchResult.IsError) + { + throw new InvalidDataException(searchResult.Error); + } + Block block = searchResult.Object; + TxReceipt txReceipt = _receiptStorage.Get(block).ForTransaction(txHash); + return block?.Transactions[txReceipt.Index]; + } + public GethLikeTxTrace GetTransactionTrace(Hash256 transactionHash, CancellationToken cancellationToken, GethTraceOptions gethTraceOptions = null) { return _tracer.Trace(transactionHash, gethTraceOptions ?? GethTraceOptions.Default, cancellationToken); @@ -153,11 +178,28 @@ public IReadOnlyCollection GetBlockTrace(Rlp blockRlp, Cancella return _tracer.TraceBlock(blockRlp, gethTraceOptions ?? GethTraceOptions.Default, cancellationToken); } + + public byte[]? GetBlockRlp(BlockParameter parameter) + { + if (parameter.BlockHash is Hash256 hash) + { + return GetBlockRlp(hash); + + } + if (parameter.BlockNumber is long num) + { + return GetBlockRlp(num); + } + return null; + } + public byte[] GetBlockRlp(Hash256 blockHash) { return _dbMappings[DbNames.Blocks].Get(blockHash); } + public Block? GetBlock(BlockParameter param) + => _blockTree.FindBlock(param); public byte[] GetBlockRlp(long number) { Hash256 hash = _blockTree.FindHash(number); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs index 4b2ba933ff5..d24f73cb7fc 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/DebugRpcModule.cs @@ -208,7 +208,7 @@ public ResultWrapper debug_gcStats() public ResultWrapper debug_getBlockRlp(long blockNumber) { - byte[] rlp = _debugBridge.GetBlockRlp(blockNumber); + byte[] rlp = _debugBridge.GetBlockRlp(new BlockParameter(blockNumber)); if (rlp is null) { return ResultWrapper.Fail($"Block {blockNumber} was not found", ErrorCodes.ResourceNotFound); @@ -219,7 +219,7 @@ public ResultWrapper debug_getBlockRlp(long blockNumber) public ResultWrapper debug_getBlockRlpByHash(Hash256 hash) { - byte[] rlp = _debugBridge.GetBlockRlp(hash); + byte[] rlp = _debugBridge.GetBlockRlp(new BlockParameter(hash)); if (rlp is null) { return ResultWrapper.Fail($"Block {hash} was not found", ErrorCodes.ResourceNotFound); @@ -261,6 +261,50 @@ public ResultWrapper debug_resetHead(Hash256 blockHash) return ResultWrapper.Success(true); } + public ResultWrapper debug_getRawTransaction(Hash256 transactionHash) + { + var transaction = _debugBridge.GetTransactionFromHash(transactionHash); + if (transaction == null) + { + return ResultWrapper.Fail($"Transaction {transactionHash} was not found", ErrorCodes.ResourceNotFound); + } + var rlp = Rlp.Encode(transaction); + return ResultWrapper.Success(rlp.Bytes); + } + + public ResultWrapper debug_getRawReceipts(long blockNumber) + { + var receipts = _debugBridge.GetReceiptsForBlock(new BlockParameter(blockNumber)); + if (receipts == null) + { + return ResultWrapper.Fail($"Receipts are not found for block {blockNumber}", ErrorCodes.ResourceNotFound); + } + + var rlp = receipts.Select(tx => Rlp.Encode(tx, RlpBehaviors.Eip658Receipts).Bytes); + return ResultWrapper.Success(rlp.ToArray()); + } + + public ResultWrapper debug_getRawBlock(long blockNumber) + { + var blockRLP = _debugBridge.GetBlockRlp(new BlockParameter(blockNumber)); + if (blockRLP == null) + { + return ResultWrapper.Fail($"Block {blockNumber} was not found", ErrorCodes.ResourceNotFound); + } + return ResultWrapper.Success(blockRLP); + } + + public ResultWrapper debug_getRawHeader(long blockNumber) + { + var block = _debugBridge.GetBlock(new BlockParameter(blockNumber)); + if (block == null) + { + return ResultWrapper.Fail($"Block {blockNumber} was not found", ErrorCodes.ResourceNotFound); + } + Rlp rlp = Rlp.Encode(block.Header); + return ResultWrapper.Success(rlp.Bytes); + } + public Task> debug_getSyncStage() { return ResultWrapper.Success(_debugBridge.GetCurrentSyncStage()); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugBridge.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugBridge.cs index 984f6b20d34..732dcdff7f7 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugBridge.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugBridge.cs @@ -22,6 +22,8 @@ public interface IDebugBridge GethLikeTxTrace? GetTransactionTrace(Transaction transaction, BlockParameter blockParameter, CancellationToken cancellationToken, GethTraceOptions? gethTraceOptions = null); IReadOnlyCollection GetBlockTrace(BlockParameter blockParameter, CancellationToken cancellationToken, GethTraceOptions gethTraceOptions = null); IReadOnlyCollection GetBlockTrace(Rlp blockRlp, CancellationToken cancellationToken, GethTraceOptions? gethTraceOptions = null); + byte[] GetBlockRlp(BlockParameter param); + Block? GetBlock(BlockParameter param); byte[] GetBlockRlp(Hash256 blockHash); byte[] GetBlockRlp(long number); byte[] GetDbValue(string dbName, byte[] key); @@ -33,4 +35,6 @@ public interface IDebugBridge void InsertReceipts(BlockParameter blockParameter, TxReceipt[] receipts); SyncReportSymmary GetCurrentSyncStage(); IEnumerable TraceBlockToFile(Hash256 blockHash, CancellationToken cancellationToken, GethTraceOptions? gethTraceOptions = null); + TxReceipt[]? GetReceiptsForBlock(BlockParameter param); + Transaction? GetTransactionFromHash(Hash256 hash); } diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs index 073dda6b020..932a1d300da 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/DebugModule/IDebugRpcModule.cs @@ -88,6 +88,18 @@ public interface IDebugRpcModule : IRpcModule [JsonRpcMethod(Description = "Insert receipts for the block after verifying receipts root correctness.")] Task> debug_insertReceipts(BlockParameter blockParameter, ReceiptForRpc[] receiptForRpc); + [JsonRpcMethod(Description = "Get Raw Block format.")] + ResultWrapper debug_getRawBlock(long blockNumber); + + [JsonRpcMethod(Description = "Get Raw Receipt format.")] + ResultWrapper debug_getRawReceipts(long blockNumber); + + [JsonRpcMethod(Description = "Get Raw Header format.")] + ResultWrapper debug_getRawHeader(long blockNumber); + + [JsonRpcMethod(Description = "Get Raw Transaction format.")] + ResultWrapper debug_getRawTransaction(Hash256 transactionHash); + [JsonRpcMethod(Description = "Retrives Nethermind Sync Stage, With extra Metadata")] Task> debug_getSyncStage(); diff --git a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs index afd4212cd7d..547df1d13ff 100644 --- a/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs +++ b/src/Nethermind/Nethermind.JsonRpc/Modules/Eth/EthRpcModule.cs @@ -388,7 +388,7 @@ public Task> eth_getTransactionByHash(Hash256 t TxReceipt receipt = null; // note that if transaction is pending then for sure no receipt is known if (transaction is null) { - (receipt, transaction, baseFee) = _blockchainBridge.GetTransaction(transactionHash); + (receipt, transaction, baseFee) = _blockchainBridge.GetTransaction(transactionHash, checkTxnPool: false); if (transaction is null) { return Task.FromResult(ResultWrapper.Success(null)); @@ -602,7 +602,9 @@ public ResultWrapper> eth_getLogs(Filter filter) SearchResult toBlockResult; if (filter.FromBlock == filter.ToBlock) + { fromBlockResult = toBlockResult = _blockFinder.SearchForHeader(filter.ToBlock); + } else { toBlockResult = _blockFinder.SearchForHeader(filter.ToBlock); @@ -625,8 +627,11 @@ public ResultWrapper> eth_getLogs(Filter filter) cancellationToken.ThrowIfCancellationRequested(); - long fromBlockNumber = fromBlockResult.Object!.Number; - long toBlockNumber = toBlockResult.Object!.Number; + BlockHeader fromBlock = fromBlockResult.Object!; + BlockHeader toBlock = toBlockResult.Object!; + + long fromBlockNumber = fromBlock.Number; + long toBlockNumber = toBlock.Number; if (fromBlockNumber > toBlockNumber && toBlockNumber != 0) { @@ -637,8 +642,9 @@ public ResultWrapper> eth_getLogs(Filter filter) try { - IEnumerable filterLogs = _blockchainBridge.GetLogs(filter.FromBlock!, filter.ToBlock!, - filter.Address, filter.Topics, cancellationToken); + LogFilter logFilter = _blockchainBridge.GetFilter(filter.FromBlock, filter.ToBlock, + filter.Address, filter.Topics); + IEnumerable filterLogs = _blockchainBridge.GetLogs(logFilter, fromBlock, toBlock, cancellationToken); return ResultWrapper>.Success(GetLogs(filterLogs, cancellationTokenSource)); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs index de7f0a7f2ec..f0729dff70a 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs @@ -20,6 +20,7 @@ using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Serialization.Rlp; using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Specs.Forks; using Nethermind.Synchronization.Blocks; using Nethermind.Synchronization.Peers; @@ -76,7 +77,7 @@ public void Can_suggest_terminal_block_correctly() .WithSpecProvider(specProvider) .OfChainLength(10) .TestObject; - PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), tree, specProvider, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), tree, specProvider, new ChainSpec(), LimboLogs.Instance); Block? block8 = tree.FindBlock(8, BlockTreeLookupOptions.None); Assert.False(block8!.IsTerminalBlock(specProvider)); @@ -95,7 +96,7 @@ public void Suggest_terminal_block_with_lower_number_and_lower_total_difficulty( .WithSpecProvider(specProvider) .OfChainLength(10) .TestObject; - PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), tree, specProvider, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), tree, specProvider, new ChainSpec(), LimboLogs.Instance); Block? block7 = tree.FindBlock(7, BlockTreeLookupOptions.None); Block newTerminalBlock = Build.A.Block @@ -124,7 +125,7 @@ public void Cannot_change_best_suggested_to_terminal_block_after_merge_block() .OfChainLength(10) .TestObject; - PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), tree, specProvider, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), tree, specProvider, new ChainSpec(), LimboLogs.Instance); Block? block8 = tree.FindBlock(8, BlockTreeLookupOptions.None); Assert.False(block8!.Header.IsTerminalBlock(specProvider)); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs index 982c5a1741b..25b5bfe2f0f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs @@ -20,6 +20,7 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Specs; using Nethermind.Core.Test.Blockchain; +using Nethermind.Core.Test.Builders; using Nethermind.Core.Timers; using Nethermind.Crypto; using Nethermind.Db; @@ -33,6 +34,7 @@ using Nethermind.Merge.Plugin.Handlers; using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Specs.Forks; using Nethermind.State; using Nethermind.Synchronization.ParallelSync; @@ -236,7 +238,7 @@ protected override IBlockProcessor CreateBlockProcessor() protected IBlockValidator CreateBlockValidator() { IBlockCacheService blockCacheService = new BlockCacheService(); - PoSSwitcher = new PoSSwitcher(MergeConfig, SyncConfig.Default, new MemDb(), BlockTree, SpecProvider, LogManager); + PoSSwitcher = new PoSSwitcher(MergeConfig, SyncConfig.Default, new MemDb(), BlockTree, SpecProvider, new ChainSpec() { Genesis = Core.Test.Builders.Build.A.Block.WithDifficulty(0).TestObject }, LogManager); SealValidator = new MergeSealValidator(PoSSwitcher, Always.Valid); HeaderValidator preMergeHeaderValidator = new HeaderValidator(BlockTree, SealValidator, SpecProvider, LogManager); HeaderValidator = new MergeHeaderValidator(PoSSwitcher, preMergeHeaderValidator, BlockTree, SpecProvider, SealValidator, LogManager); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs index 335b0707324..ab33e79b4f0 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Synchronization.cs @@ -77,6 +77,8 @@ public async Task forkChoiceUpdatedV1_unknown_block_initiates_syncing() chain.BeaconSync.IsBeaconSyncHeadersFinished().Should().BeFalse(); chain.BeaconSync.IsBeaconSyncFinished(chain.BlockTree.FindBlock(block.Hash!)?.Header).Should().BeFalse(); AssertBeaconPivotValues(chain.BeaconPivot, block.Header); + + block.Header.TotalDifficulty = 0; pointers.LowestInsertedBeaconHeader = block.Header; pointers.BestKnownBeaconBlock = block.Number; pointers.LowestInsertedHeader = block.Header; @@ -151,6 +153,7 @@ public async Task forkChoiceUpdatedV1_unknown_block_parent_while_syncing_initiat chain.BeaconSync.IsBeaconSyncFinished(chain.BlockTree.FindBlock(block.Hash!)?.Header).Should().BeFalse(); AssertBeaconPivotValues(chain.BeaconPivot, block.Header); + block.Header.TotalDifficulty = 0; pointers.LowestInsertedBeaconHeader = block.Header; pointers.BestKnownBeaconBlock = block.Number; pointers.LowestInsertedHeader = block.Header; @@ -167,6 +170,7 @@ public async Task forkChoiceUpdatedV1_unknown_block_parent_while_syncing_initiat chain.BeaconSync.IsBeaconSyncFinished(chain.BlockTree.FindBlock(nextUnconnectedBlock.Hash!)?.Header).Should().BeFalse(); AssertBeaconPivotValues(chain.BeaconPivot, nextUnconnectedBlock.Header); + nextUnconnectedBlock.Header.TotalDifficulty = 0; pointers.LowestInsertedBeaconHeader = nextUnconnectedBlock.Header; pointers.BestKnownBeaconBlock = nextUnconnectedBlock.Number; AssertBlockTreePointers(chain.BlockTree, pointers); @@ -967,7 +971,7 @@ private void AssertBeaconPivotValues(IBeaconPivot beaconPivot, BlockHeader block beaconPivot.BeaconPivotExists().Should().BeTrue(); beaconPivot.PivotNumber.Should().Be(blockHeader.Number); beaconPivot.PivotHash.Should().Be(blockHeader.Hash ?? blockHeader.CalculateHash()); - beaconPivot.PivotTotalDifficulty.Should().Be(null); + beaconPivot.PivotTotalDifficulty.Should().Be((UInt256)0); } private class BlockTreePointers diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/MergeRewardCalculatorTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/MergeRewardCalculatorTests.cs index 8175a677e5f..3c49333bdaa 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/MergeRewardCalculatorTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/MergeRewardCalculatorTests.cs @@ -10,6 +10,7 @@ using Nethermind.Logging; using Nethermind.Merge.Plugin.Handlers; using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Specs.Forks; using NSubstitute; using NUnit.Framework; @@ -156,7 +157,7 @@ private static PoSSwitcher CreatePosSwitcher() specProvider.TerminalTotalDifficulty = 2; MergeConfig? mergeConfig = new() { }; IBlockCacheService blockCacheService = new BlockCacheService(); - return new PoSSwitcher(mergeConfig, new SyncConfig(), db, blockTree, specProvider, LimboLogs.Instance); + return new PoSSwitcher(mergeConfig, new SyncConfig(), db, blockTree, specProvider, new ChainSpec(), LimboLogs.Instance); } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/PoSSwitcherTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/PoSSwitcherTests.cs index c76c2a59063..1c1b5838b21 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/PoSSwitcherTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/PoSSwitcherTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.IO; +using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; @@ -29,7 +30,7 @@ public void Initial_TTD_should_be_null() { UInt256? expectedTtd = null; IBlockTree blockTree = Substitute.For(); - PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), blockTree, TestSpecProvider.Instance, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), blockTree, TestSpecProvider.Instance, new ChainSpec(), LimboLogs.Instance); Assert.That(poSSwitcher.TerminalTotalDifficulty, Is.EqualTo(expectedTtd)); } @@ -44,7 +45,7 @@ public void Read_TTD_from_chainspec_if_not_specified_in_merge_config() ChainSpec chainSpec = loader.Load(File.ReadAllText(path)); ChainSpecBasedSpecProvider specProvider = new(chainSpec); - PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), blockTree, specProvider, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig(), new SyncConfig(), new MemDb(), blockTree, specProvider, new ChainSpec(), LimboLogs.Instance); Assert.That(poSSwitcher.TerminalTotalDifficulty, Is.EqualTo(expectedTtd)); Assert.That(specProvider.MergeBlockNumber?.BlockNumber, Is.EqualTo(101)); @@ -92,7 +93,7 @@ public void Override_TTD_and_number_from_merge_config() IBlockTree blockTree = Substitute.For(); TestSpecProvider specProvider = new(London.Instance); specProvider.UpdateMergeTransitionInfo(100, 20); - PoSSwitcher poSSwitcher = new(new MergeConfig() { TerminalTotalDifficulty = "340", TerminalBlockNumber = 2000 }, new SyncConfig(), new MemDb(), blockTree, specProvider, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig() { TerminalTotalDifficulty = "340", TerminalBlockNumber = 2000 }, new SyncConfig(), new MemDb(), blockTree, specProvider, new ChainSpec(), LimboLogs.Instance); Assert.That(poSSwitcher.TerminalTotalDifficulty, Is.EqualTo(expectedTtd)); Assert.That(specProvider.MergeBlockNumber?.BlockNumber, Is.EqualTo(2001)); @@ -105,7 +106,7 @@ public void Can_update_merge_transition_info() IBlockTree blockTree = Substitute.For(); TestSpecProvider specProvider = new(London.Instance); specProvider.UpdateMergeTransitionInfo(2001, expectedTtd); - PoSSwitcher poSSwitcher = new(new MergeConfig() { }, new SyncConfig(), new MemDb(), blockTree, specProvider, LimboLogs.Instance); + PoSSwitcher poSSwitcher = new(new MergeConfig() { }, new SyncConfig(), new MemDb(), blockTree, specProvider, new ChainSpec(), LimboLogs.Instance); Assert.That(poSSwitcher.TerminalTotalDifficulty, Is.EqualTo(expectedTtd)); Assert.That(specProvider.MergeBlockNumber?.BlockNumber, Is.EqualTo(2001)); @@ -222,11 +223,50 @@ public void Can_load_parameters_after_the_restart() Assert.That(newPoSSwitcher.HasEverReachedTerminalBlock(), Is.EqualTo(true)); } + [Test] + public void No_final_difficulty_if_conditions_are_not_met() + { + AssertFinalTotalDifficulty(10005, 10000, 10000, null); + } + + [TestCase(0, 1)] + [TestCase(0, 0)] + [TestCase(5000, 6000)] + public void Can_set_final_total_difficulty_for_post_merge_networks(long ttd, long genesisDifficulty) + { + AssertFinalTotalDifficulty(ttd, genesisDifficulty, null, genesisDifficulty); + } + + [TestCase(0, 1)] + [TestCase(0, 0)] + [TestCase(5000, 6000)] + public void Can_set_final_total_difficulty_based_on_sync_pivot(long ttd, long pivotTotalDifficulty) + { + AssertFinalTotalDifficulty(ttd, 0, pivotTotalDifficulty, pivotTotalDifficulty); + } + + private void AssertFinalTotalDifficulty(long ttd, long genesisDifficulty, long? pivotTotalDifficulty, long? expectedFinalTotalDifficulty) + { + TestSpecProvider specProvider = new(London.Instance); + specProvider.TerminalTotalDifficulty = (UInt256)ttd; + Block genesisBlock = Build.A.Block.WithNumber(0).WithDifficulty((UInt256)genesisDifficulty).TestObject; + BlockTree blockTree = Build.A.BlockTree(genesisBlock, specProvider).OfChainLength(4).TestObject; + SyncConfig syncConfig = new(); + if (pivotTotalDifficulty != null) + syncConfig = new SyncConfig() { PivotTotalDifficulty = $"{(UInt256)pivotTotalDifficulty}" }; + PoSSwitcher poSSwitcher = new PoSSwitcher(new MergeConfig(), syncConfig, new MemDb(), blockTree, specProvider, new ChainSpec() { Genesis = genesisBlock }, LimboLogs.Instance); + if (expectedFinalTotalDifficulty != null) + poSSwitcher.FinalTotalDifficulty.Should().Be((UInt256)expectedFinalTotalDifficulty); + else + poSSwitcher.FinalTotalDifficulty.Should().BeNull(); + } + + private static PoSSwitcher CreatePosSwitcher(IBlockTree blockTree, IDb? db = null, ISpecProvider? specProvider = null) { db ??= new MemDb(); MergeConfig? mergeConfig = new() { }; - return new PoSSwitcher(mergeConfig, new SyncConfig(), db, blockTree, specProvider ?? MainnetSpecProvider.Instance, LimboLogs.Instance); + return new PoSSwitcher(mergeConfig, new SyncConfig(), db, blockTree, specProvider ?? MainnetSpecProvider.Instance, new ChainSpec(), LimboLogs.Instance); } } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs new file mode 100644 index 00000000000..8e0b754b1a8 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Db; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Specs; +using Nethermind.TxPool; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Merge.Plugin.Test; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ProcessedTransactionsDbCleanerTests +{ + private readonly ILogManager _logManager = LimboLogs.Instance; + private readonly ISpecProvider _specProvider = MainnetSpecProvider.Instance; + + [Test] + public async Task should_remove_processed_txs_from_db_after_finalization([Values(0, 1, 42, 358)] long blockOfTxs, [Values(1, 42, 358)] long finalizedBlock) + { + Transaction GetTx(PrivateKey sender) + { + return Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(new EthereumEcdsa(_specProvider.ChainId, _logManager), sender).TestObject; + } + + IColumnsDb columnsDb = new MemColumnsDb(BlobTxsColumns.ProcessedTxs); + BlobTxStorage blobTxStorage = new(columnsDb); + Transaction[] txs = { GetTx(TestItem.PrivateKeyA), GetTx(TestItem.PrivateKeyB) }; + + blobTxStorage.AddBlobTransactionsFromBlock(blockOfTxs, txs); + + blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out Transaction[]? returnedTxs).Should().BeTrue(); + returnedTxs!.Length.Should().Be(2); + + IBlockFinalizationManager finalizationManager = Substitute.For(); + ProcessedTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _logManager); + + finalizationManager.BlocksFinalized += Raise.EventWith( + new FinalizeEventArgs(Build.A.BlockHeader.TestObject, + Build.A.BlockHeader.WithNumber(finalizedBlock).TestObject)); + + await dbCleaner.CleaningTask; + + blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out returnedTxs).Should().Be(blockOfTxs > finalizedBlock); + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs index 8006941d498..1d9bdcbd1ae 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/Synchronization/BeaconHeadersSyncTests.cs @@ -18,6 +18,7 @@ using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.State.Repositories; using Nethermind.Synchronization; using Nethermind.Synchronization.FastBlocks; @@ -71,7 +72,7 @@ public IBeaconPivot BeaconPivot private PoSSwitcher? _poSSwitcher; public PoSSwitcher PoSSwitcher => _poSSwitcher ??= new(MergeConfig, SyncConfig, MetadataDb, BlockTree, - MainnetSpecProvider.Instance, LimboLogs.Instance); + MainnetSpecProvider.Instance, new ChainSpec(), LimboLogs.Instance); private IInvalidChainTracker? _invalidChainTracker; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs b/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs index 17d736dd280..5e22b0d1fe7 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs @@ -28,22 +28,16 @@ public GCKeeper(IGCStrategy gcStrategy, ILogManager logManager) public IDisposable TryStartNoGCRegion(long? size = null) { - // SustainedLowLatency is used rather than NoGCRegion - // due to runtime bug https://github.com/dotnet/runtime/issues/84096 - // The code is left in as comments so it can be reverted when the bug is fixed size ??= _defaultSize; - var priorLatencyMode = System.Runtime.GCSettings.LatencyMode; - //if (_gcStrategy.CanStartNoGCRegion()) - if (priorLatencyMode != GCLatencyMode.SustainedLowLatency) + if (_gcStrategy.CanStartNoGCRegion()) { FailCause failCause = FailCause.None; try { - GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency; - //if (!System.GC.TryStartNoGCRegion(size.Value, true)) - //{ - // failCause = FailCause.GCFailedToStartNoGCRegion; - //} + if (!System.GC.TryStartNoGCRegion(size.Value, true)) + { + failCause = FailCause.GCFailedToStartNoGCRegion; + } } catch (ArgumentOutOfRangeException) { @@ -60,10 +54,10 @@ public IDisposable TryStartNoGCRegion(long? size = null) if (_logger.IsError) _logger.Error($"{nameof(System.GC.TryStartNoGCRegion)} failed with exception.", e); } - return new NoGCRegion(this, priorLatencyMode, failCause, size, _logger); + return new NoGCRegion(this, failCause, size, _logger); } - return new NoGCRegion(this, priorLatencyMode, FailCause.StrategyDisallowed, size, _logger); + return new NoGCRegion(this, FailCause.StrategyDisallowed, size, _logger); } private enum FailCause @@ -79,15 +73,13 @@ private enum FailCause private class NoGCRegion : IDisposable { private readonly GCKeeper _gcKeeper; - private readonly GCLatencyMode _priorMode; private readonly FailCause _failCause; private readonly long? _size; private readonly ILogger _logger; - internal NoGCRegion(GCKeeper gcKeeper, GCLatencyMode priorMode, FailCause failCause, long? size, ILogger logger) + internal NoGCRegion(GCKeeper gcKeeper, FailCause failCause, long? size, ILogger logger) { _gcKeeper = gcKeeper; - _priorMode = priorMode; _failCause = failCause; _size = size; _logger = logger; @@ -95,19 +87,13 @@ internal NoGCRegion(GCKeeper gcKeeper, GCLatencyMode priorMode, FailCause failCa public void Dispose() { - // SustainedLowLatency is used rather than NoGCRegion - // due to runtime bug https://github.com/dotnet/runtime/issues/84096 - // The code is left in as comments so it can be reverted when the bug is fixed if (_failCause == FailCause.None) { - //if (GCSettings.LatencyMode == GCLatencyMode.NoGCRegion) - if (GCSettings.LatencyMode == GCLatencyMode.SustainedLowLatency && - _priorMode != GCLatencyMode.SustainedLowLatency) + if (GCSettings.LatencyMode == GCLatencyMode.NoGCRegion) { try { - GCSettings.LatencyMode = _priorMode; - //System.GC.EndNoGCRegion(); + System.GC.EndNoGCRegion(); _gcKeeper.ScheduleGC(); } catch (InvalidOperationException) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs index d8ec1a33370..2685c2981bb 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/ForkchoiceUpdatedHandler.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Blockchain.Find; @@ -12,6 +13,7 @@ using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; +using Nethermind.Core.Threading; using Nethermind.Crypto; using Nethermind.JsonRpc; using Nethermind.Logging; @@ -86,6 +88,8 @@ public Task> Handle(ForkchoiceStateV1 f private ResultWrapper? ApplyForkchoiceUpdate(Block? newHeadBlock, ForkchoiceStateV1 forkchoiceState, PayloadAttributes? payloadAttributes) { + using var handle = Thread.CurrentThread.BoostPriority(); + if (_invalidChainTracker.IsOnKnownInvalidChain(forkchoiceState.HeadBlockHash, out Hash256? lastValidHash)) { if (_logger.IsInfo) _logger.Info($"Received Invalid {forkchoiceState} {payloadAttributes} - {forkchoiceState.HeadBlockHash} is known to be a part of an invalid chain."); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index a8f65248086..eebb8fc2b17 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading; @@ -16,7 +15,6 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Validators; using Nethermind.Core; -using Nethermind.Core.Crypto; using Nethermind.Core.Exceptions; using Nethermind.Db; using Nethermind.Facade.Proxy; @@ -26,14 +24,13 @@ using Nethermind.Logging; using Nethermind.Merge.Plugin.BlockProduction; using Nethermind.Merge.Plugin.BlockProduction.Boost; -using Nethermind.Merge.Plugin.Data; using Nethermind.Merge.Plugin.GC; using Nethermind.Merge.Plugin.Handlers; using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Reporting; using Nethermind.Trie.Pruning; +using Nethermind.TxPool; namespace Nethermind.Merge.Plugin; @@ -44,6 +41,7 @@ public partial class MergePlugin : IConsensusWrapperPlugin, ISynchronizationPlug protected IMergeConfig _mergeConfig = null!; private ISyncConfig _syncConfig = null!; protected IBlocksConfig _blocksConfig = null!; + protected ITxPoolConfig _txPoolConfig = null!; protected IPoSSwitcher _poSSwitcher = NoPoS.Instance; private IBeaconPivot? _beaconPivot; private BeaconSync? _beaconSync; @@ -70,6 +68,7 @@ public virtual Task Init(INethermindApi nethermindApi) _mergeConfig = nethermindApi.Config(); _syncConfig = nethermindApi.Config(); _blocksConfig = nethermindApi.Config(); + _txPoolConfig = nethermindApi.Config(); MigrateSecondsPerSlot(_blocksConfig, _mergeConfig); @@ -93,6 +92,7 @@ public virtual Task Init(INethermindApi nethermindApi) _api.DbProvider.GetDb(DbNames.Metadata), _api.BlockTree, _api.SpecProvider, + _api.ChainSpec, _api.LogManager); _invalidChainTracker = new InvalidChainTracker.InvalidChainTracker( _poSSwitcher, @@ -102,6 +102,11 @@ public virtual Task Init(INethermindApi nethermindApi) _api.PoSSwitcher = _poSSwitcher; _api.DisposeStack.Push(_invalidChainTracker); _blockFinalizationManager = new ManualBlockFinalizationManager(); + if (_txPoolConfig.BlobsSupport.SupportsReorgs()) + { + ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); + _api.DisposeStack.Push(processedTransactionsDbCleaner); + } _api.RewardCalculatorSource = new MergeRewardCalculatorSource( _api.RewardCalculatorSource ?? NoBlockRewards.Instance, _poSSwitcher); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/PoSSwitcher.cs b/src/Nethermind/Nethermind.Merge.Plugin/PoSSwitcher.cs index 492bbc20c12..632d4716a8f 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/PoSSwitcher.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/PoSSwitcher.cs @@ -12,6 +12,7 @@ using Nethermind.Db; using Nethermind.Logging; using Nethermind.Serialization.Rlp; +using Nethermind.Specs.ChainSpecStyle; namespace Nethermind.Merge.Plugin { @@ -39,6 +40,7 @@ public class PoSSwitcher : IPoSSwitcher private readonly IDb _metadataDb; private readonly IBlockTree _blockTree; private readonly ISpecProvider _specProvider; + private readonly ChainSpec _chainSpec; private readonly ILogger _logger; private Hash256? _terminalBlockHash; @@ -55,6 +57,7 @@ public PoSSwitcher( IDb metadataDb, IBlockTree blockTree, ISpecProvider specProvider, + ChainSpec chainSpec, ILogManager logManager) { _mergeConfig = mergeConfig; @@ -62,6 +65,7 @@ public PoSSwitcher( _metadataDb = metadataDb; _blockTree = blockTree; _specProvider = specProvider; + _chainSpec = chainSpec; _logger = logManager.GetClassLogger(); Initialize(); @@ -88,11 +92,24 @@ private void LoadFinalTotalDifficulty() { _finalTotalDifficulty = _mergeConfig.FinalTotalDifficultyParsed; + if (TerminalTotalDifficulty is null) + return; + // pivot post TTD, so we know FinalTotalDifficulty - if (_syncConfig.PivotTotalDifficultyParsed != 0 && TerminalTotalDifficulty is not null && _syncConfig.PivotTotalDifficultyParsed >= TerminalTotalDifficulty) + if (_syncConfig.PivotTotalDifficultyParsed != 0 && _syncConfig.PivotTotalDifficultyParsed >= TerminalTotalDifficulty) { _finalTotalDifficulty = _syncConfig.PivotTotalDifficultyParsed; } + else + { + if (_chainSpec?.Genesis == null) return; + + UInt256 genesisDifficulty = _chainSpec.Genesis.Difficulty; + if (genesisDifficulty >= TerminalTotalDifficulty) // networks with the merge in genesis + { + _finalTotalDifficulty = genesisDifficulty; + } + } } private void CheckIfTerminalBlockReached(object? sender, BlockEventArgs e) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs new file mode 100644 index 00000000000..7babdf12c72 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Threading.Tasks; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Db; +using Nethermind.Logging; + +namespace Nethermind.Merge.Plugin; + +public class ProcessedTransactionsDbCleaner : IDisposable +{ + private readonly IBlockFinalizationManager _finalizationManager; + private readonly IDb _processedTxsDb; + private readonly ILogger _logger; + private long _lastFinalizedBlock = 0; + public Task CleaningTask { get; private set; } = Task.CompletedTask; + + public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) + { + _finalizationManager = finalizationManager ?? throw new ArgumentNullException(nameof(finalizationManager)); + _processedTxsDb = processedTxsDb ?? throw new ArgumentNullException(nameof(processedTxsDb)); + _logger = logManager.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + + _finalizationManager.BlocksFinalized += OnBlocksFinalized; + } + + private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) + { + if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > _lastFinalizedBlock) + { + CleaningTask = Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); + } + } + + private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) + { + try + { + using (IWriteBatch writeBatch = _processedTxsDb.StartWriteBatch()) + { + foreach (byte[] key in _processedTxsDb.GetAllKeys()) + { + long blockNumber = key.ToLongFromBigEndianByteArrayWithoutLeadingZeros(); + if (newlyFinalizedBlockNumber >= blockNumber) + { + if (_logger.IsTrace) _logger.Trace($"Cleaning processed blob txs from block {blockNumber}"); + writeBatch.Delete(blockNumber); + } + } + } + + if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); + + _lastFinalizedBlock = newlyFinalizedBlockNumber; + } + catch (Exception exception) + { + if (_logger.IsError) _logger.Error($"Couldn't correctly clean db with processed transactions. Newly finalized block {newlyFinalizedBlockNumber}, last finalized block: {_lastFinalizedBlock}", exception); + } + } + + public void Dispose() + { + _finalizationManager.BlocksFinalized -= OnBlocksFinalized; + } +} diff --git a/src/Nethermind/Nethermind.Mining.Test/MinGasPriceTests.cs b/src/Nethermind/Nethermind.Mining.Test/MinGasPriceTests.cs index 2e613d8c8e9..bd1e9e4db65 100644 --- a/src/Nethermind/Nethermind.Mining.Test/MinGasPriceTests.cs +++ b/src/Nethermind/Nethermind.Mining.Test/MinGasPriceTests.cs @@ -54,6 +54,19 @@ public void Test1559(long minimum, long maxFeePerGas, long maxPriorityFeePerGas, specProvider.GetSpec(Arg.Any(), Arg.Any()).IsEip1559Enabled.Returns(true); specProvider.GetSpec(Arg.Any()).IsEip1559Enabled.Returns(true); specProvider.GetSpec(Arg.Any()).IsEip1559Enabled.Returns(true); + + specProvider.GetSpec(Arg.Any()).ForkBaseFee.Returns(Eip1559Constants.DefaultForkBaseFee); + specProvider.GetSpec(Arg.Any()).BaseFeeMaxChangeDenominator.Returns(Eip1559Constants.DefaultBaseFeeMaxChangeDenominator); + specProvider.GetSpec(Arg.Any()).ElasticityMultiplier.Returns(Eip1559Constants.DefaultElasticityMultiplier); + + specProvider.GetSpec(Arg.Any(), Arg.Any()).ForkBaseFee.Returns(Eip1559Constants.DefaultForkBaseFee); + specProvider.GetSpec(Arg.Any(), Arg.Any()).BaseFeeMaxChangeDenominator.Returns(Eip1559Constants.DefaultBaseFeeMaxChangeDenominator); + specProvider.GetSpec(Arg.Any(), Arg.Any()).ElasticityMultiplier.Returns(Eip1559Constants.DefaultElasticityMultiplier); + + specProvider.GetSpec(Arg.Any()).ForkBaseFee.Returns(Eip1559Constants.DefaultForkBaseFee); + specProvider.GetSpec(Arg.Any()).BaseFeeMaxChangeDenominator.Returns(Eip1559Constants.DefaultBaseFeeMaxChangeDenominator); + specProvider.GetSpec(Arg.Any()).ElasticityMultiplier.Returns(Eip1559Constants.DefaultElasticityMultiplier); + BlocksConfig blocksConfig = new() { MinGasPrice = (UInt256)minimum diff --git a/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs b/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs index f7d9e87423a..08203506e87 100644 --- a/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs +++ b/src/Nethermind/Nethermind.Monitoring/MonitoringService.cs @@ -81,13 +81,7 @@ public async Task StartAsync() } if (_exposePort is not null) { - IMetricServer metricServer = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? - // MetricServer uses HttpListener which on Windows either needs - // permissions at OS or Admin mode, whereas KestrelMetricServer doesn't need those - new KestrelMetricServer(_exposePort.Value) : - // KestrelMetricServer intercept SIGTERM causing exitcode to be incorrect - new MetricServer(_exposePort.Value); - metricServer.Start(); + new NethermindKestrelMetricServer(_exposePort.Value).Start(); } await Task.Factory.StartNew(() => _metricsController.StartUpdating(), TaskCreationOptions.LongRunning); if (_logger.IsInfo) _logger.Info($"Started monitoring for the group: {_options.Group}, instance: {_options.Instance}"); diff --git a/src/Nethermind/Nethermind.Monitoring/NethermindKestrelMetricServer.cs b/src/Nethermind/Nethermind.Monitoring/NethermindKestrelMetricServer.cs new file mode 100644 index 00000000000..0fe8c582f2b --- /dev/null +++ b/src/Nethermind/Nethermind.Monitoring/NethermindKestrelMetricServer.cs @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +#nullable enable +using System; +using System.Net; +using System.Security.Cryptography.X509Certificates; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.Extensions.DependencyInjection; +using Prometheus; + +namespace Nethermind.Monitoring; + +/// +/// Copy of KestrelMetricServer but does not wait for Ctrl-C so that it does not intercept exit code +/// +public sealed class NethermindKestrelMetricServer : MetricHandler +{ + public NethermindKestrelMetricServer(int port, string url = "/metrics", CollectorRegistry? registry = null, X509Certificate2? certificate = null) : this("+", port, url, registry, certificate) + { + } + + public NethermindKestrelMetricServer(string hostname, int port, string url = "/metrics", CollectorRegistry? registry = null, X509Certificate2? certificate = null) : this(LegacyOptions(hostname, port, url, registry, certificate)) + { + } + + private static KestrelMetricServerOptions LegacyOptions(string hostname, int port, string url, CollectorRegistry? registry, X509Certificate2? certificate) => + new KestrelMetricServerOptions + { + Hostname = hostname, + Port = (ushort)port, + Url = url, + Registry = registry, + TlsCertificate = certificate, + }; + + public NethermindKestrelMetricServer(KestrelMetricServerOptions options) + { + _hostname = options.Hostname; + _port = options.Port; + _url = options.Url; + _certificate = options.TlsCertificate; + + // We use one callback to apply the legacy settings, and from within this we call the real callback. + _configureExporter = settings => + { + // Legacy setting, may be overridden by ConfigureExporter. + settings.Registry = options.Registry; + + if (options.ConfigureExporter != null) + options.ConfigureExporter(settings); + }; + } + + private readonly string _hostname; + private readonly int _port; + private readonly string _url; + + private readonly X509Certificate2? _certificate; + + private readonly Action _configureExporter; + + protected override Task StartServer(CancellationToken cancel) + { + var s = _certificate != null ? "s" : ""; + var hostAddress = $"http{s}://{_hostname}:{_port}"; + + // If the caller needs to customize any of this, they can just set up their own web host and inject the middleware. + var builder = new WebHostBuilder() + .UseKestrel() + .UseIISIntegration() + .Configure(app => + { + app.UseMetricServer(_configureExporter, _url); + + // If there is any URL prefix, we just redirect people going to root URL to our prefix. + if (!string.IsNullOrWhiteSpace(_url.Trim('/'))) + { + app.MapWhen(context => context.Request.Path.Value?.Trim('/') == "", + configuration => + { + configuration.Use((HttpContext context, RequestDelegate next) => + { + context.Response.Redirect(_url); + return Task.CompletedTask; + }); + }); + } + }); + + if (_certificate != null) + { + builder = builder.ConfigureServices(services => + { + Action configureEndpoint = options => + { + options.UseHttps(_certificate); + }; + + services.Configure(options => + { + options.Listen(IPAddress.Any, _port, configureEndpoint); + }); + }); + } + else + { + builder = builder.UseUrls(hostAddress); + } + + var webHost = builder.Build(); + + // This is what changed + // webHost.Start(); + // return webHost.WaitForShutdownAsync(cancel); + + return webHost.RunAsync(cancel); + } +} diff --git a/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs b/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs index ab5b8daa3ee..ac26dcd4253 100644 --- a/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/ForkInfoTests.cs @@ -107,8 +107,10 @@ public void Fork_id_and_hash_as_expected_with_merge_fork_id(long head, ulong hea [TestCase(4_600_000L, 0ul, "0x757a1c47", 5_062_605ul, "Future Berlin block")] [TestCase(5_062_605L, 0ul, "0xB8C6299D", 1678832736ul, "First London block")] [TestCase(6_000_000, 0ul, "0xB8C6299D", 1678832736ul, "Future London block")] - [TestCase(6_000_001, 1678832736ul, "0xf9843abf", 0ul, "First Shanghai timestamp")] - [TestCase(6_000_001, 2678832736ul, "0xf9843abf", 0ul, "Future Shanghai timestamp")] + [TestCase(6_000_001, 1678832736ul, "0xf9843abf", 1705473120ul, "First Shanghai timestamp")] + [TestCase(6_000_001, 1705473119ul, "0xf9843abf", 1705473120ul, "Future Shanghai timestamp")] + [TestCase(6_000_002, 1705473120ul, "0x70cc14e2", 0ul, "First Cancun timestamp")] + [TestCase(6_000_002, 1905473119ul, "0x70cc14e2", 0ul, "Future Cancun timestamp")] public void Fork_id_and_hash_as_expected_on_goerli(long head, ulong headTimestamp, string forkHashHex, ulong next, string description) { Test(head, headTimestamp, KnownHashes.GoerliGenesis, forkHashHex, next, description, GoerliSpecProvider.Instance, "goerli.json"); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs index 3aa898824dc..ae3bd356a7f 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs @@ -94,7 +94,7 @@ public void RequestTransactionsEth68(Action Always.Valid; protected override IHealthHintService CreateHealthHintService() => diff --git a/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs b/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs index d9c56cf389c..3902d88b024 100644 --- a/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs +++ b/src/Nethermind/Nethermind.Optimism/OPConfigHelper.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; +using Nethermind.Specs.ChainSpecStyle; namespace Nethermind.Optimism; @@ -9,14 +10,18 @@ public class OPSpecHelper : IOPConfigHelper { private readonly ulong _regolithTimestamp; private readonly long _bedrockBlockNumber; + private readonly ulong? _canyonTimestamp; public Address L1FeeReceiver { get; init; } - public OPSpecHelper(ulong regolithTimestamp, long bedrockBlockNumber, Address l1FeeReceiver) + public OPSpecHelper(OptimismParameters parameters) { - _regolithTimestamp = regolithTimestamp; - _bedrockBlockNumber = bedrockBlockNumber; - L1FeeReceiver = l1FeeReceiver; + _regolithTimestamp = parameters.RegolithTimestamp; + _bedrockBlockNumber = parameters.BedrockBlockNumber; + _canyonTimestamp = parameters.CanyonTimestamp; + L1FeeReceiver = parameters.L1FeeRecipient; + Create2DeployerCode = parameters.Create2DeployerCode; + Create2DeployerAddress = parameters.Create2DeployerAddress; } public bool IsRegolith(BlockHeader header) @@ -28,4 +33,12 @@ public bool IsBedrock(BlockHeader header) { return header.Number >= _bedrockBlockNumber; } + + public bool IsCanyon(BlockHeader header) + { + return header.Timestamp >= (_canyonTimestamp ?? long.MaxValue); + } + + public Address? Create2DeployerAddress { get; } + public byte[]? Create2DeployerCode { get; } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs new file mode 100644 index 00000000000..50389d91351 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockProcessor.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Processing; +using Nethermind.Consensus.Rewards; +using Nethermind.Consensus.Validators; +using Nethermind.Consensus.Withdrawals; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Evm.Tracing; +using Nethermind.Logging; +using Nethermind.State; + +namespace Nethermind.Optimism; + +public class OptimismBlockProcessor : BlockProcessor +{ + private Create2DeployerContractRewriter? _contractRewriter; + + public OptimismBlockProcessor( + ISpecProvider? specProvider, + IBlockValidator? blockValidator, + IRewardCalculator? rewardCalculator, + IBlockProcessor.IBlockTransactionsExecutor? blockTransactionsExecutor, + IWorldState? stateProvider, + IReceiptStorage? receiptStorage, + IWitnessCollector? witnessCollector, + ILogManager? logManager, + IOPConfigHelper opConfigHelper, + Create2DeployerContractRewriter contractRewriter, + IWithdrawalProcessor? withdrawalProcessor = null) + : base(specProvider, blockValidator, rewardCalculator, blockTransactionsExecutor, + stateProvider, receiptStorage, witnessCollector, logManager, withdrawalProcessor) + { + ArgumentNullException.ThrowIfNull(stateProvider); + _contractRewriter = contractRewriter; + ReceiptsTracer = new OptimismBlockReceiptTracer(opConfigHelper, stateProvider); + } + + protected override TxReceipt[] ProcessBlock(Block block, IBlockTracer blockTracer, ProcessingOptions options) + { + _contractRewriter?.RewriteContract(block.Header, _stateProvider); + return base.ProcessBlock(block, blockTracer, options); + } +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs index 7393e7fdfa8..42df6a18aa3 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockProducerEnvFactory.cs @@ -10,14 +10,11 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Transactions; using Nethermind.Consensus.Validators; -using Nethermind.Core; +using Nethermind.Consensus.Withdrawals; using Nethermind.Core.Specs; -using Nethermind.Db; -using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.Specs.ChainSpecStyle; using Nethermind.State; -using Nethermind.Trie.Pruning; using Nethermind.TxPool; namespace Nethermind.Optimism; @@ -74,4 +71,26 @@ protected override ITxSource CreateTxSourceForProducer(ITxSource? additionalTxSo return new OptimismTxPoolTxSource(baseTxSource); } + + protected override BlockProcessor CreateBlockProcessor( + ReadOnlyTxProcessingEnv readOnlyTxProcessingEnv, + ISpecProvider specProvider, + IBlockValidator blockValidator, + IRewardCalculatorSource rewardCalculatorSource, + IReceiptStorage receiptStorage, + ILogManager logManager, + IBlocksConfig blocksConfig) + { + return new OptimismBlockProcessor(specProvider, + blockValidator, + rewardCalculatorSource.Get(readOnlyTxProcessingEnv.TransactionProcessor), + TransactionsExecutorFactory.Create(readOnlyTxProcessingEnv), + readOnlyTxProcessingEnv.StateProvider, + receiptStorage, + NullWitnessCollector.Instance, + logManager, + _specHelper, + new Create2DeployerContractRewriter(_specHelper, _specProvider, _blockTree), + new BlockProductionWithdrawalProcessor(new WithdrawalProcessor(readOnlyTxProcessingEnv.StateProvider, logManager))); + } } diff --git a/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs b/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs new file mode 100644 index 00000000000..95048f27080 --- /dev/null +++ b/src/Nethermind/Nethermind.Optimism/OptimismBlockReceiptTracer.cs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Evm.Tracing; +using Nethermind.State; + +namespace Nethermind.Optimism; + +public class OptimismBlockReceiptTracer : BlockReceiptsTracer +{ + private IOPConfigHelper _opConfigHelper; + private IWorldState _worldState; + + public OptimismBlockReceiptTracer(IOPConfigHelper opConfigHelper, IWorldState worldState) + { + _opConfigHelper = opConfigHelper; + _worldState = worldState; + } + + private (ulong?, ulong?) GetDepositReceiptData(BlockHeader header) + { + ArgumentNullException.ThrowIfNull(CurrentTx); + + ulong? depositNonce = null; + ulong? version = null; + + if (CurrentTx.IsDeposit()) + { + depositNonce = _worldState.GetNonce(CurrentTx.SenderAddress!).ToUInt64(null); + if (_opConfigHelper.IsCanyon(header)) + { + version = 1; + } + } + + return (depositNonce == 0 ? 0 : depositNonce - 1, version); + } + + protected override TxReceipt BuildReceipt(Address recipient, long spentGas, byte statusCode, LogEntry[] logEntries, Hash256? stateRoot) + { + (ulong? depositNonce, ulong? version) = GetDepositReceiptData(Block.Header); + TxReceipt receipt = base.BuildReceipt(recipient, spentGas, statusCode, logEntries, stateRoot); + receipt.DepositNonce = depositNonce; + receipt.DepositReceiptVersion = version; + return receipt; + } +} diff --git a/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs b/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs index b58391a4317..ac0f4c4f2e4 100644 --- a/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs +++ b/src/Nethermind/Nethermind.Optimism/OptimismTransactionProcessor.cs @@ -33,11 +33,10 @@ public OptimismTransactionProcessor( protected override void Execute(Transaction tx, BlockExecutionContext blCtx, ITxTracer tracer, ExecutionOptions opts) { + IReleaseSpec spec = SpecProvider.GetSpec(blCtx.Header); _currentTxL1Cost = null; if (tx.IsDeposit()) { - IReleaseSpec spec = SpecProvider.GetSpec(blCtx.Header); - WorldState.AddToBalanceAndCreateIfNotExists(tx.SenderAddress!, tx.Mint, spec); if (opts.HasFlag(ExecutionOptions.Commit) || !spec.IsEip658Enabled) diff --git a/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs b/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs index 41c11c8608b..f6ed7ececdd 100644 --- a/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs @@ -189,11 +189,12 @@ public void Network_diag_tracer_disabled_by_default(string configWildcard) } [TestCase("mainnet", 2048)] + [TestCase("holesky", 2048)] [TestCase("gnosis", 2048)] [TestCase("poacore", 2048)] [TestCase("energy", 2048)] [TestCase("chiado", 2048)] - [TestCase("^mainnet ^spaceneth ^volta ^energy ^poacore ^gnosis ^chiado", 1024)] + [TestCase("^mainnet ^holesky ^spaceneth ^volta ^energy ^poacore ^gnosis ^chiado", 1024)] [TestCase("spaceneth", 128)] public void Tx_pool_defaults_are_correct(string configWildcard, int poolSize) { @@ -205,8 +206,9 @@ public void Tx_pool_defaults_are_correct(string configWildcard, int poolSize) [TestCase("gnosis", true)] [TestCase("mainnet", true)] [TestCase("sepolia", true)] + [TestCase("holesky", true)] [TestCase("chiado", true)] - [TestCase("^spaceneth ^goerli ^mainnet ^gnosis ^sepolia ^chiado", false)] + [TestCase("^spaceneth ^goerli ^mainnet ^gnosis ^sepolia ^holesky ^chiado", false)] public void Json_defaults_are_correct(string configWildcard, bool jsonEnabled) { Test(configWildcard, c => c.Enabled, jsonEnabled); @@ -241,10 +243,11 @@ public void Snap_sync_settings_as_expected(string configWildcard, bool enabled) Test(configWildcard, c => c.SnapSync, enabled); } - [TestCase("^aura ^sepolia ^goerli ^mainnet", false)] + [TestCase("^aura ^sepolia ^holesky ^goerli ^mainnet", false)] [TestCase("aura ^archive", true)] [TestCase("^archive ^spaceneth", true)] [TestCase("sepolia ^archive", true)] + [TestCase("holesky ^archive", true)] [TestCase("goerli ^archive", true)] [TestCase("mainnet ^archive", true)] public void Stays_on_full_sync(string configWildcard, bool stickToFullSyncAfterFastSync) @@ -323,6 +326,18 @@ public void Basic_configs_are_as_expected(string configWildcard, bool isProducti Test(configWildcard, c => c.LogFileName, (cf, p) => p.Should().Be(cf.Replace("cfg", "logs.txt"), cf)); } + [TestCase("goerli", BlobsSupportMode.StorageWithReorgs)] + [TestCase("^goerli", BlobsSupportMode.Disabled)] + [TestCase("sepolia", BlobsSupportMode.Disabled)] + [TestCase("holesky", BlobsSupportMode.Disabled)] + [TestCase("mainnet", BlobsSupportMode.Disabled)] + [TestCase("chiado", BlobsSupportMode.Disabled)] + [TestCase("gnosis", BlobsSupportMode.Disabled)] + public void Blob_txs_support_is_correct(string configWildcard, BlobsSupportMode blobsSupportMode) + { + Test(configWildcard, c => c.BlobsSupport, blobsSupportMode); + } + [TestCase("goerli", new[] { 16, 16, 16, 16 })] [TestCase("mainnet")] @@ -357,7 +372,8 @@ public void Arena_order_is_default(string configWildcard) [TestCase("goerli", 30_000_000L)] [TestCase("mainnet", 30_000_000L)] [TestCase("sepolia", 30_000_000L)] - [TestCase("^chiado ^gnosis ^goerli ^mainnet ^sepolia")] + [TestCase("holesky", 30_000_000L)] + [TestCase("^chiado ^gnosis ^goerli ^mainnet ^sepolia ^holesky")] public void Blocks_defaults_are_correct(string configWildcard, long? targetBlockGasLimit = null, ulong secondsPerSlot = 12) { Test(configWildcard, c => c.TargetBlockGasLimit, targetBlockGasLimit); @@ -408,8 +424,8 @@ public void Memory_hint_is_enough(string configWildcard) { "goerli_archive.cfg", "goerli.cfg", - "kovan.cfg", - "kovan_archive.cfg", + "holesky.cfg", + "holesky_archive.cfg", "mainnet_archive.cfg", "mainnet.cfg", "poacore.cfg", @@ -420,8 +436,6 @@ public void Memory_hint_is_enough(string configWildcard) "spaceneth_persistent.cfg", "volta.cfg", "volta_archive.cfg", - "volta.cfg", - "volta_archive.cfg", "energyweb.cfg", "energyweb_archive.cfg", "sepolia.cfg", diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs index 8762884e529..fe452fb0691 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/Migrations/ReceiptMigrationTests.cs @@ -161,7 +161,9 @@ public void EnsureCanonical(Block block) { } - public event EventHandler ReceiptsInserted { add { } remove { } } +#pragma warning disable CS0067 + public event EventHandler ReceiptsInserted; +#pragma warning restore CS0067 } } } diff --git a/src/Nethermind/Nethermind.Runner/Program.cs b/src/Nethermind/Nethermind.Runner/Program.cs index 6f44d203ae0..699d32d7edb 100644 --- a/src/Nethermind/Nethermind.Runner/Program.cs +++ b/src/Nethermind/Nethermind.Runner/Program.cs @@ -11,6 +11,7 @@ using System.Runtime.InteropServices; using System.Runtime.Loader; using System.Text; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; #if !DEBUG @@ -56,6 +57,8 @@ public static class Program public static void Main(string[] args) { + // Increase regex cache size as more added in log coloring matches + Regex.CacheSize = 128; #if !DEBUG ResourceLeakDetector.Level = ResourceLeakDetector.DetectionLevel.Disabled; #endif diff --git a/src/Nethermind/Nethermind.Runner/configs/chiado.cfg b/src/Nethermind/Nethermind.Runner/configs/chiado.cfg index 1767febe2c6..bf562a4a091 100644 --- a/src/Nethermind/Nethermind.Runner/configs/chiado.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/chiado.cfg @@ -13,8 +13,8 @@ "Sync": { "FastSync": true, "FastBlocks": true, - "PivotNumber": 7380000, - "PivotHash": "0xcb9bb3e1d8bc0c7e6b12fcfac1f5c874d6bf352984c9da5959e227ec21d985d6", + "PivotNumber": 7580000, + "PivotHash": "0xf23063619522a038100cb2fc66e7947294e6dd6da0422ff769f2833729f7d6fb", "PivotTotalDifficulty": "231708131825107706987652208063906496124457284", "FastSyncCatchUpHeightDelta": "10000000000", "UseGethLimitsInFastBlocks": false diff --git a/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg b/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg index 3ee86b0cf02..4d49d044dbb 100644 --- a/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/energyweb.cfg @@ -8,9 +8,9 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 27490000, - "PivotHash": "0x65659c40c76d8c349ac7dbc0188c2e20e244660eb9ebedde46dd8b879516061d", - "PivotTotalDifficulty": "9354362266656598360608167958299308132557544671", + "PivotNumber": 27710000, + "PivotHash": "0xf5f417b63a2a43ba7fcc610017f23380db4969600a8f7f2672ad774713247cd2", + "PivotTotalDifficulty": "9429224387379204822570110371934297139077403490", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, "FastSyncCatchUpHeightDelta": 10000000000 diff --git a/src/Nethermind/Nethermind.Runner/configs/exosama.cfg b/src/Nethermind/Nethermind.Runner/configs/exosama.cfg index e548ebcbd63..81c11ed694a 100644 --- a/src/Nethermind/Nethermind.Runner/configs/exosama.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/exosama.cfg @@ -8,9 +8,9 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 7260000, - "PivotHash": "0x3dcbd46634e84cda6cb200592fa375e301293a155e036af289db9cddd513e3da", - "PivotTotalDifficulty": "2470449983846013244744099649954637214822893288", + "PivotNumber": 7510000, + "PivotHash": "0x77046fb443f3c3f0d3a168a35800fb1fe577100b41965f579b658e9c70e6a0a1", + "PivotTotalDifficulty": "2555520575576247860609943301812579267686393288", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, "FastSyncCatchUpHeightDelta": 10000000000 diff --git a/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg b/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg index 51accfc5267..6ded43f6d74 100644 --- a/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/gnosis.cfg @@ -12,8 +12,8 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 31460000, - "PivotHash": "0x98692140a13606c7e3f8a2d3a30dd76622d8bb25cae82d18af5df5fc67a555b4", + "PivotNumber": 31690000, + "PivotHash": "0xb1c5f676cb85d4794c0ee949bc44adbee925da08d982136c2813c4e5f4b56768", "PivotTotalDifficulty": "8626000110427538733349499292577475819600160930", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, diff --git a/src/Nethermind/Nethermind.Runner/configs/goerli.cfg b/src/Nethermind/Nethermind.Runner/configs/goerli.cfg index 7b3880a355e..17931707434 100644 --- a/src/Nethermind/Nethermind.Runner/configs/goerli.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/goerli.cfg @@ -7,7 +7,8 @@ "MemoryHint": 768000000 }, "TxPool": { - "Size": 1024 + "Size": 1024, + "BlobsSupport": "StorageWithReorgs" }, "Db": { "EnableMetricsUpdater": true @@ -15,8 +16,8 @@ "Sync": { "FastSync": true, "SnapSync": true, - "PivotNumber": 10200000, - "PivotHash": "0x014d2d7336c3c4543eaea77e4edd43edc8a900ab42b2bf1317c7095a01824078", + "PivotNumber": 10260000, + "PivotHash": "0xfd7265da4de69c506d16eb67eb41018c4dd11e3a0a8198cee0c5308e655b61db", "PivotTotalDifficulty": "10790000", "FastBlocks": true, "UseGethLimitsInFastBlocks": true, diff --git a/src/Nethermind/Nethermind.Runner/configs/goerli_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/goerli_archive.cfg index f3778061e48..5f8f89a5c75 100644 --- a/src/Nethermind/Nethermind.Runner/configs/goerli_archive.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/goerli_archive.cfg @@ -7,7 +7,8 @@ "MemoryHint": 768000000 }, "TxPool": { - "Size": 1024 + "Size": 1024, + "BlobsSupport": "StorageWithReorgs" }, "EthStats": { "Server": "wss://stats.goerli.net/api", diff --git a/src/Nethermind/Nethermind.Runner/configs/holesky.cfg b/src/Nethermind/Nethermind.Runner/configs/holesky.cfg index b0409121834..819128ed83c 100644 --- a/src/Nethermind/Nethermind.Runner/configs/holesky.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/holesky.cfg @@ -8,7 +8,8 @@ "Sync": { "FastSync": true, "SnapSync": true, - "FastBlocks": true + "FastBlocks": true, + "FastSyncCatchUpHeightDelta": "10000000000" }, "Metrics": { "NodeName": "Holesky" @@ -26,7 +27,6 @@ "EngineEnabledModules": "net,eth,subscribe,engine,web3,client" }, "Merge": { - "Enabled": true, - "FinalTotalDifficulty": "1" + "Enabled": true } } diff --git a/src/Nethermind/Nethermind.Runner/configs/holesky_archive.cfg b/src/Nethermind/Nethermind.Runner/configs/holesky_archive.cfg index bc7aa9030d3..eb2e77a02a9 100644 --- a/src/Nethermind/Nethermind.Runner/configs/holesky_archive.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/holesky_archive.cfg @@ -27,7 +27,6 @@ "EngineEnabledModules": "net,eth,subscribe,engine,web3,client" }, "Merge": { - "Enabled": true, - "FinalTotalDifficulty": "1" + "Enabled": true } } diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg index 752435ebf37..5a2153deff0 100644 --- a/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/joc-mainnet.cfg @@ -9,9 +9,9 @@ "FastSync": true, "SnapSync": true, "FastBlocks": true, - "PivotNumber": 8630000, - "PivotHash": "0x265b02f054ccc44e8adfd3b7b0c0dec420fc35bf2c58c8e688de7e4c006e7eca", - "PivotTotalDifficulty": "17259058" + "PivotNumber": 8870000, + "PivotHash": "0x3d8c071530dd409324d5b74916f94f1553afb850051ae6143dd9bdbd4ad2b866", + "PivotTotalDifficulty": "17734268" }, "Metrics": { "NodeName": "JOC-Mainnet" diff --git a/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg b/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg index 2b66a605fe1..b78abb7f41b 100644 --- a/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/joc-testnet.cfg @@ -9,9 +9,9 @@ "FastSync": true, "SnapSync": true, "FastBlocks": true, - "PivotNumber": 2230000, - "PivotHash": "0x58a6f20c2d6bf1149bf49f7a70694fadd7d15e259eccb7de3fa31c3f4a9bbba7", - "PivotTotalDifficulty": "4459816" + "PivotNumber": 2470000, + "PivotHash": "0x457e4a8f96f002fd503c380c8b45b077a9280b3f00b6198cf80cd69f22c67b59", + "PivotTotalDifficulty": "4939802" }, "Metrics": { "NodeName": "JOC-Testnet" diff --git a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg index 53480f88a4b..1c5a8cca358 100644 --- a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg @@ -12,8 +12,8 @@ "Sync": { "FastSync": true, "SnapSync": true, - "PivotNumber": 18801000, - "PivotHash": "0xbdab3335cb22c0e2ca2786dd046720f854461de0c7eca8b1880a9d563328e860", + "PivotNumber": 18900000, + "PivotHash": "0x84bd4976f299d9343a18a1088f1cec0dbcac72b6d51a6fc9b901921a55449fc6", "PivotTotalDifficulty": "58750003716598352816469", "FastBlocks": true, "FastSyncCatchUpHeightDelta": "10000000000" diff --git a/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg b/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg index cd9dbfdb686..c0aff1477e2 100644 --- a/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/sepolia.cfg @@ -18,8 +18,8 @@ "FastBlocks": true, "SnapSync": true, "UseGethLimitsInFastBlocks": true, - "PivotNumber": 4899000, - "PivotHash": "0xa94b70d0af991d47cb67b22027137835e2ed7b3e08bb8ec00766491ed9b75f7b", + "PivotNumber": 4988000, + "PivotHash": "0xfa0e1cff9cf2ad67171d4c4b2e19b09f411f79756fd99d861744607dd88e9131", "PivotTotalDifficulty": "17000018015853232", "FastSyncCatchUpHeightDelta": "10000000000" }, diff --git a/src/Nethermind/Nethermind.Runner/configs/volta.cfg b/src/Nethermind/Nethermind.Runner/configs/volta.cfg index 4da1540cffb..f6685fcce6b 100644 --- a/src/Nethermind/Nethermind.Runner/configs/volta.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/volta.cfg @@ -12,9 +12,9 @@ }, "Sync": { "FastSync": true, - "PivotNumber": 25920000, - "PivotHash": "0x7959742aefd389555fbc8a9e2d64768c2d748d887ecec5597f220fee914929ce", - "PivotTotalDifficulty": "8820118950590724972970669824631432040573188062", + "PivotNumber": 26100000, + "PivotHash": "0xbd8ddbf4f3677a4e959f961478f56a971816f275b443a51387ba68e623cc4aec", + "PivotTotalDifficulty": "8881369776636493896394077253969150318634850916", "FastBlocks": true, "UseGethLimitsInFastBlocks": false, "FastSyncCatchUpHeightDelta": 10000000000 diff --git a/src/Nethermind/Nethermind.Serialization.Json/DoubleConverter.cs b/src/Nethermind/Nethermind.Serialization.Json/DoubleConverter.cs index d62578904da..044a661a8d2 100644 --- a/src/Nethermind/Nethermind.Serialization.Json/DoubleConverter.cs +++ b/src/Nethermind/Nethermind.Serialization.Json/DoubleConverter.cs @@ -5,7 +5,7 @@ namespace Nethermind.Serialization.Json { - using System.Collections.Generic; + using Nethermind.Core.Collections; using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.Json.Serialization; @@ -47,19 +47,21 @@ public override double[] Read( { throw new JsonException(); } - List values = null; - reader.Read(); - while (reader.TokenType == JsonTokenType.Number) + using ArrayPoolList values = new ArrayPoolList(16); + while (reader.Read() && reader.TokenType == JsonTokenType.Number) { - values ??= new List(); values.Add(reader.GetDouble()); } if (reader.TokenType != JsonTokenType.EndArray) { throw new JsonException(); } - reader.Read(); - return values?.ToArray() ?? Array.Empty(); + + if (values.Count == 0) return Array.Empty(); + + double[] result = new double[values.Count]; + values.CopyTo(result, 0); + return result; } [SkipLocalsInit] diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs index 052a6e766b1..ce58b0c0c95 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs @@ -57,8 +57,14 @@ public TxReceipt Decode(RlpStream rlpStream, RlpBehaviors rlpBehaviors = RlpBeha { entries[i] = Rlp.Decode(rlpStream, RlpBehaviors.AllowExtraBytes); } - txReceipt.Logs = entries; + + if (txReceipt.TxType == TxType.DepositTx && rlpStream.Position != lastCheck) + { + txReceipt.DepositNonce = rlpStream.DecodeUlong(); + txReceipt.DepositReceiptVersion = rlpStream.DecodeUlong(); + } + return txReceipt; } @@ -85,6 +91,12 @@ private static (int Total, int Logs) GetContentLength(TxReceipt item, RlpBehavio : Rlp.LengthOf(item.PostTransactionState); } + if (item.TxType == TxType.DepositTx && item.DepositReceiptVersion is not null) + { + contentLength += Rlp.LengthOf(item.DepositNonce ?? 0); + contentLength += Rlp.LengthOf(item.DepositReceiptVersion.Value); + } + return (contentLength, logsLength); } @@ -173,6 +185,12 @@ public void Encode(RlpStream rlpStream, TxReceipt item, RlpBehaviors rlpBehavior { rlpStream.Encode(item.Logs[i]); } + + if (item.TxType == TxType.DepositTx && item.DepositReceiptVersion is not null) + { + rlpStream.Encode(item.DepositNonce!.Value); + rlpStream.Encode(item.DepositReceiptVersion.Value); + } } } } diff --git a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs index 681839f3248..1dd79cb0a19 100644 --- a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecBasedSpecProviderTests.cs @@ -58,11 +58,8 @@ public void Timstamp_activation_equal_to_genesis_timestamp_loads_correctly(long testProvider.SpecToReturn = expectedSpec; testProvider.TerminalTotalDifficulty = 0; testProvider.GenesisSpec = expectedSpec; - List forkActivationsToTest = new() - { - (blockNumber, timestamp), - }; - CompareSpecProviders(testProvider, provider, forkActivationsToTest); + + CompareSpecs(testProvider, provider, (blockNumber, timestamp)); Assert.That(provider.GenesisSpec.Eip1559TransitionBlock, Is.EqualTo(testProvider.GenesisSpec.Eip1559TransitionBlock)); Assert.That(provider.GenesisSpec.DifficultyBombDelay, Is.EqualTo(testProvider.GenesisSpec.DifficultyBombDelay)); } @@ -121,23 +118,26 @@ public void Logs_warning_when_timestampActivation_happens_before_blockActivation } } - [Test] - public void Sepolia_loads_properly() + public static IEnumerable SepoliaActivations + { + get + { + yield return new TestCaseData((ForkActivation)(2, 0)) { TestName = "First" }; + yield return new TestCaseData((ForkActivation)(120_000_000, 0)) { TestName = "Block number" }; + yield return new TestCaseData((ForkActivation)(1735372, 3)) { TestName = "Low timestamp" }; + yield return new TestCaseData((ForkActivation)(1735372, 1677557088)) { TestName = "1677557088" }; + yield return new TestCaseData((ForkActivation)(1735372, 1677557087)) { TestName = "1677557087" }; + } + } + + [TestCaseSource(nameof(SepoliaActivations))] + public void Sepolia_loads_properly(ForkActivation forkActivation) { ChainSpec chainSpec = LoadChainSpecFromChainFolder("sepolia"); ChainSpecBasedSpecProvider provider = new(chainSpec); SepoliaSpecProvider sepolia = SepoliaSpecProvider.Instance; - List forkActivationsToTest = new() - { - new ForkActivation(2, 0), - new ForkActivation(120_000_000, 0), - new ForkActivation(1735372, 3), - new ForkActivation(1735372, 1677557088), - new ForkActivation(1735372, 1677557087) - }; - - CompareSpecProviders(sepolia, provider, forkActivationsToTest); + CompareSpecs(sepolia, provider, forkActivation); Assert.That(provider.TerminalTotalDifficulty, Is.EqualTo(SepoliaSpecProvider.Instance.TerminalTotalDifficulty)); Assert.That(provider.GenesisSpec.Eip1559TransitionBlock, Is.EqualTo(0)); Assert.That(provider.GenesisSpec.DifficultyBombDelay, Is.EqualTo(long.MaxValue)); @@ -148,23 +148,26 @@ public void Sepolia_loads_properly() t => ValidateSlotByTimestamp(t, SepoliaSpecProvider.BeaconChainGenesisTimestamp).Should().BeTrue()); } - [Test] - public void Holesky_loads_properly() + public static IEnumerable HoleskyActivations { - ChainSpec chainSpec = LoadChainSpecFromChainFolder("holesky"); - ChainSpecBasedSpecProvider provider = new(chainSpec); - ISpecProvider hardCodedSpec = HoleskySpecProvider.Instance; - - List forkActivationsToTest = new() + get { - new ForkActivation(0, HoleskySpecProvider.GenesisTimestamp), - new ForkActivation(1, HoleskySpecProvider.ShanghaiTimestamp), - new ForkActivation(3, HoleskySpecProvider.ShanghaiTimestamp + 24), + yield return new TestCaseData(new ForkActivation(0, HoleskySpecProvider.GenesisTimestamp)) { TestName = "Genesis" }; + yield return new TestCaseData(new ForkActivation(1, HoleskySpecProvider.ShanghaiTimestamp)) { TestName = "Shanghai" }; + yield return new TestCaseData(new ForkActivation(3, HoleskySpecProvider.ShanghaiTimestamp + 24)) { TestName = "Post Shanghai" }; //new ForkActivation(4, HoleskySpecProvider.CancunTimestamp), //new ForkActivation(5, HoleskySpecProvider.CancunTimestamp + 12), - }; + } + } - CompareSpecProviders(hardCodedSpec, provider, forkActivationsToTest); + [TestCaseSource(nameof(HoleskyActivations))] + public void Holesky_loads_properly(ForkActivation forkActivation) + { + ChainSpec chainSpec = LoadChainSpecFromChainFolder("holesky"); + ChainSpecBasedSpecProvider provider = new(chainSpec); + ISpecProvider hardCodedSpec = HoleskySpecProvider.Instance; + + CompareSpecs(hardCodedSpec, provider, forkActivation); Assert.That(provider.TerminalTotalDifficulty, Is.EqualTo(hardCodedSpec.TerminalTotalDifficulty)); Assert.That(provider.GenesisSpec.Eip1559TransitionBlock, Is.EqualTo(0)); Assert.That(provider.GenesisSpec.DifficultyBombDelay, Is.EqualTo(0)); @@ -176,28 +179,34 @@ public void Holesky_loads_properly() // t => ValidateSlotByTimestamp(t, HoleskySpecProvider.GenesisTimestamp).Should().BeTrue()); } - [Test] - public void Goerli_loads_properly() + public static IEnumerable GoerliActivations + { + get + { + yield return new TestCaseData((ForkActivation)0) { TestName = "Genesis" }; + yield return new TestCaseData((ForkActivation)1) { TestName = "1" }; + yield return new TestCaseData((ForkActivation)(GoerliSpecProvider.IstanbulBlockNumber - 1)) { TestName = "Before Istanbul" }; + yield return new TestCaseData((ForkActivation)GoerliSpecProvider.IstanbulBlockNumber) { TestName = "Istanbul" }; + yield return new TestCaseData((ForkActivation)(GoerliSpecProvider.BerlinBlockNumber - 1)) { TestName = "Before Berlin" }; + yield return new TestCaseData((ForkActivation)GoerliSpecProvider.BerlinBlockNumber) { TestName = "Berlin" }; + yield return new TestCaseData((ForkActivation)(GoerliSpecProvider.LondonBlockNumber - 1)) { TestName = "Before London" }; + yield return new TestCaseData((ForkActivation)GoerliSpecProvider.LondonBlockNumber) { TestName = "London" }; + yield return new TestCaseData(new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 1, GoerliSpecProvider.ShanghaiTimestamp - 1)) { TestName = "Before Shanghai" }; + yield return new TestCaseData(new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 1, GoerliSpecProvider.ShanghaiTimestamp)) { TestName = "Shanghai" }; + yield return new TestCaseData(new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 2, GoerliSpecProvider.CancunTimestamp - 1)) { TestName = "Before Cancun" }; + yield return new TestCaseData(new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 2, GoerliSpecProvider.CancunTimestamp)) { TestName = "Cancun" }; + yield return new TestCaseData(new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 2, GoerliSpecProvider.CancunTimestamp + 100000000)) { TestName = "Future" }; + } + } + + [TestCaseSource(nameof(GoerliActivations))] + public void Goerli_loads_properly(ForkActivation forkActivation) { ChainSpec chainSpec = LoadChainSpecFromChainFolder("goerli"); ChainSpecBasedSpecProvider provider = new(chainSpec); GoerliSpecProvider goerli = GoerliSpecProvider.Instance; - List forkActivationsToTest = new() - { - (ForkActivation)0, - (ForkActivation)1, - (ForkActivation)(GoerliSpecProvider.IstanbulBlockNumber - 1), - (ForkActivation)GoerliSpecProvider.IstanbulBlockNumber, - (ForkActivation)(GoerliSpecProvider.BerlinBlockNumber - 1), - (ForkActivation)GoerliSpecProvider.BerlinBlockNumber, - (ForkActivation)(GoerliSpecProvider.LondonBlockNumber - 1), - (ForkActivation)GoerliSpecProvider.LondonBlockNumber, - new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 1, GoerliSpecProvider.ShanghaiTimestamp), - new ForkActivation(GoerliSpecProvider.LondonBlockNumber + 1, GoerliSpecProvider.ShanghaiTimestamp + 100000000) // far in future - }; - - CompareSpecProviders(goerli, provider, forkActivationsToTest); + CompareSpecs(goerli, provider, forkActivation); Assert.That(provider.GenesisSpec.Eip1559TransitionBlock, Is.EqualTo(GoerliSpecProvider.LondonBlockNumber)); Assert.That(provider.TerminalTotalDifficulty, Is.EqualTo(GoerliSpecProvider.Instance.TerminalTotalDifficulty)); Assert.That(provider.ChainId, Is.EqualTo(BlockchainIds.Goerli)); @@ -207,23 +216,26 @@ public void Goerli_loads_properly() t => ValidateSlotByTimestamp(t, GoerliSpecProvider.BeaconChainGenesisTimestamp).Should().BeTrue()); } - [Test] - public void Chiado_loads_properly() + public static IEnumerable ChiadoActivations + { + get + { + yield return new TestCaseData((ForkActivation)0) { TestName = "Genesis" }; + yield return new TestCaseData((ForkActivation)(1, 20)) { TestName = "(1, 20)" }; + yield return new TestCaseData((ForkActivation)(1, ChiadoSpecProvider.ShanghaiTimestamp - 1)) { TestName = "Before Shanghai" }; + yield return new TestCaseData((ForkActivation)(1, ChiadoSpecProvider.ShanghaiTimestamp)) { TestName = "Shanghai" }; + yield return new TestCaseData(new ForkActivation(1, GoerliSpecProvider.ShanghaiTimestamp + 100000000)) { TestName = "Future" }; + } + } + + [TestCaseSource(nameof(ChiadoActivations))] + public void Chiado_loads_properly(ForkActivation forkActivation) { ChainSpec chainSpec = LoadChainSpecFromChainFolder("chiado"); ChainSpecBasedSpecProvider provider = new(chainSpec); ChiadoSpecProvider chiado = ChiadoSpecProvider.Instance; - List forkActivationsToTest = new() - { - (ForkActivation)0, - (ForkActivation)(1, 20), - (1, ChiadoSpecProvider.ShanghaiTimestamp - 1), - (1, ChiadoSpecProvider.ShanghaiTimestamp), - (999_999_999, 999_999_999) // far in the future - }; - - CompareSpecProviders(chiado, provider, forkActivationsToTest, CompareSpecsOptions.IsGnosis); + CompareSpecs(chiado, provider, forkActivation, CompareSpecsOptions.IsGnosis); Assert.That(provider.TerminalTotalDifficulty, Is.EqualTo(ChiadoSpecProvider.Instance.TerminalTotalDifficulty)); Assert.That(provider.ChainId, Is.EqualTo(BlockchainIds.Chiado)); Assert.That(provider.NetworkId, Is.EqualTo(BlockchainIds.Chiado)); @@ -244,33 +256,36 @@ public void Chiado_loads_properly() }); } - [Test] - public void Gnosis_loads_properly() + public static IEnumerable GnosisActivations + { + get + { + yield return new TestCaseData((ForkActivation)0) { TestName = "Genesis" }; + yield return new TestCaseData((ForkActivation)1) { TestName = "Genesis + 1" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.ConstantinopoleBlockNumber - 1)) { TestName = "Before Constantinopole" }; + yield return new TestCaseData((ForkActivation)GnosisSpecProvider.ConstantinopoleBlockNumber) { TestName = "Constantinopole" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.ConstantinopoleFixBlockNumber - 1)) { TestName = "Before ConstantinopoleFix" }; + yield return new TestCaseData((ForkActivation)GnosisSpecProvider.ConstantinopoleFixBlockNumber) { TestName = "ConstantinopoleFix" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.IstanbulBlockNumber - 1)) { TestName = "Before Istanbul" }; + yield return new TestCaseData((ForkActivation)GnosisSpecProvider.IstanbulBlockNumber) { TestName = "Istanbul" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.BerlinBlockNumber - 1)) { TestName = "Before Berlin" }; + yield return new TestCaseData((ForkActivation)GnosisSpecProvider.BerlinBlockNumber) { TestName = "Berlin" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.LondonBlockNumber - 1)) { TestName = "Before London" }; + yield return new TestCaseData((ForkActivation)GnosisSpecProvider.LondonBlockNumber) { TestName = "London" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.LondonBlockNumber, GnosisSpecProvider.ShanghaiTimestamp - 1)) { TestName = "Before Shanghai" }; + yield return new TestCaseData((ForkActivation)(GnosisSpecProvider.LondonBlockNumber, GnosisSpecProvider.ShanghaiTimestamp)) { TestName = "Shanghai" }; + yield return new TestCaseData((ForkActivation)(999_999_999, 999_999_999)) { TestName = "Future" }; + } + } + + [TestCaseSource(nameof(GnosisActivations))] + public void Gnosis_loads_properly(ForkActivation forkActivation) { ChainSpec chainSpec = LoadChainSpecFromChainFolder("gnosis"); ChainSpecBasedSpecProvider provider = new(chainSpec); GnosisSpecProvider gnosisSpecProvider = GnosisSpecProvider.Instance; - List forkActivationsToTest = new() - { - (ForkActivation)0, - (ForkActivation)1, - (ForkActivation)(GnosisSpecProvider.ConstantinopoleBlockNumber -1), - (ForkActivation)(GnosisSpecProvider.ConstantinopoleBlockNumber), - (ForkActivation)(GnosisSpecProvider.ConstantinopoleFixBlockNumber -1), - (ForkActivation)(GnosisSpecProvider.ConstantinopoleFixBlockNumber), - (ForkActivation)(GnosisSpecProvider.IstanbulBlockNumber -1), - (ForkActivation)(GnosisSpecProvider.IstanbulBlockNumber), - (ForkActivation)(GnosisSpecProvider.BerlinBlockNumber -1), - (ForkActivation)(GnosisSpecProvider.BerlinBlockNumber), - (ForkActivation)(GnosisSpecProvider.LondonBlockNumber -1), - (ForkActivation)(GnosisSpecProvider.LondonBlockNumber), - (GnosisSpecProvider.LondonBlockNumber, GnosisSpecProvider.ShanghaiTimestamp - 1), - (GnosisSpecProvider.LondonBlockNumber, GnosisSpecProvider.ShanghaiTimestamp), - (999_999_999, 999_999_999) // far in the future - }; - - CompareSpecProviders(gnosisSpecProvider, provider, forkActivationsToTest, CompareSpecsOptions.IsGnosis); + CompareSpecs(gnosisSpecProvider, provider, forkActivation, CompareSpecsOptions.IsGnosis); Assert.That(provider.TerminalTotalDifficulty, Is.EqualTo(GnosisSpecProvider.Instance.TerminalTotalDifficulty)); Assert.That(provider.ChainId, Is.EqualTo(BlockchainIds.Gnosis)); Assert.That(provider.NetworkId, Is.EqualTo(BlockchainIds.Gnosis)); @@ -323,47 +338,49 @@ private void VerifyGnosisPreShanghaiExceptions(ISpecProvider specProvider) .BeTrue(); } + public static IEnumerable MainnetActivations + { + get + { + yield return new TestCaseData((ForkActivation)0) { TestName = "Genesis" }; + yield return new TestCaseData((ForkActivation)(0, null)) { TestName = "Genesis null" }; + yield return new TestCaseData((ForkActivation)(0, 0)) { TestName = "Genesis timestamp" }; + yield return new TestCaseData((ForkActivation)1) { TestName = "Genesis + 1" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.HomesteadBlockNumber - 1)) { TestName = "Before Homestead" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.HomesteadBlockNumber) { TestName = "Homestead" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.TangerineWhistleBlockNumber - 1)) { TestName = "Before TangerineWhistle" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.TangerineWhistleBlockNumber) { TestName = "TangerineWhistle" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.SpuriousDragonBlockNumber - 1)) { TestName = "Before SpuriousDragon" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.SpuriousDragonBlockNumber) { TestName = "SpuriousDragon" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.ByzantiumBlockNumber - 1)) { TestName = "Before Byzantium" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.ByzantiumBlockNumber) { TestName = "Byzantium" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.ConstantinopleFixBlockNumber - 1)) { TestName = "Before Constantinople" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.ConstantinopleFixBlockNumber) { TestName = "Constantinople" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.IstanbulBlockNumber - 1)) { TestName = "Before Istanbul" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.IstanbulBlockNumber) { TestName = "Istanbul" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.MuirGlacierBlockNumber - 1)) { TestName = "Before MuirGlacier" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.MuirGlacierBlockNumber) { TestName = "MuirGlacier" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.BerlinBlockNumber - 1)) { TestName = "Before Berlin" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.BerlinBlockNumber) { TestName = "Berlin" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.LondonBlockNumber - 1)) { TestName = "Before London" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.LondonBlockNumber) { TestName = "London" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.ArrowGlacierBlockNumber - 1)) { TestName = "Before ArrowGlacier" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.ArrowGlacierBlockNumber) { TestName = "ArrowGlacier" }; + yield return new TestCaseData((ForkActivation)(MainnetSpecProvider.ArrowGlacierBlockNumber - 1)) { TestName = "Before GrayGlacier" }; + yield return new TestCaseData((ForkActivation)MainnetSpecProvider.ArrowGlacierBlockNumber) { TestName = "GrayGlacier" }; + yield return new TestCaseData(MainnetSpecProvider.ShanghaiActivation) { TestName = "Shanghai" }; + yield return new TestCaseData(new ForkActivation(99_000_000, 99_681_338_455)) { TestName = "Future" }; + } + } - [Test] - public void Mainnet_loads_properly() + [TestCaseSource(nameof(MainnetActivations))] + public void Mainnet_loads_properly(ForkActivation forkActivation) { ChainSpec chainSpec = LoadChainSpecFromChainFolder("foundation"); ChainSpecBasedSpecProvider provider = new(chainSpec); MainnetSpecProvider mainnet = MainnetSpecProvider.Instance; - List forkActivationsToTest = new() - { - (ForkActivation)0, - (0, 0), - (0, null), - (ForkActivation)1, - (ForkActivation)(MainnetSpecProvider.HomesteadBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.HomesteadBlockNumber, - (ForkActivation)(MainnetSpecProvider.TangerineWhistleBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.TangerineWhistleBlockNumber, - (ForkActivation)(MainnetSpecProvider.SpuriousDragonBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.SpuriousDragonBlockNumber, - (ForkActivation)(MainnetSpecProvider.ByzantiumBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.ByzantiumBlockNumber, - (ForkActivation)(MainnetSpecProvider.ConstantinopleFixBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.ConstantinopleFixBlockNumber, - (ForkActivation)(MainnetSpecProvider.IstanbulBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.IstanbulBlockNumber, - (ForkActivation)(MainnetSpecProvider.MuirGlacierBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.MuirGlacierBlockNumber, - (ForkActivation)(MainnetSpecProvider.BerlinBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.BerlinBlockNumber, - (ForkActivation)(MainnetSpecProvider.LondonBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.LondonBlockNumber, - (ForkActivation)(MainnetSpecProvider.ArrowGlacierBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.ArrowGlacierBlockNumber, - (ForkActivation)(MainnetSpecProvider.GrayGlacierBlockNumber - 1), - (ForkActivation)MainnetSpecProvider.GrayGlacierBlockNumber, - MainnetSpecProvider.ShanghaiActivation, - new ForkActivation(99_000_000, 99_681_338_455) // far in the future - }; - - CompareSpecProviders(mainnet, provider, forkActivationsToTest, CompareSpecsOptions.CheckDifficultyBomb); + CompareSpecs(mainnet, provider, forkActivation, CompareSpecsOptions.CheckDifficultyBomb); provider.GetSpec((MainnetSpecProvider.SpuriousDragonBlockNumber, null)).MaxCodeSize.Should().Be(24576L); provider.GetSpec((MainnetSpecProvider.SpuriousDragonBlockNumber, null)).MaxInitCodeSize.Should().Be(2 * 24576L); @@ -401,24 +418,21 @@ enum CompareSpecsOptions IsGnosis = 4 // for Gnosis and Chiado testnets } - private static void CompareSpecProviders( + private static void CompareSpecs( ISpecProvider oldSpecProvider, ISpecProvider newSpecProvider, - IEnumerable forkActivations, + ForkActivation activation, CompareSpecsOptions compareSpecsOptions = CompareSpecsOptions.None) { - foreach (ForkActivation activation in forkActivations) - { - IReleaseSpec oldSpec = oldSpecProvider.GetSpec(activation); - IReleaseSpec newSpec = newSpecProvider.GetSpec(activation); - long? daoBlockNumber = newSpecProvider.DaoBlockNumber; + IReleaseSpec oldSpec = oldSpecProvider.GetSpec(activation); + IReleaseSpec newSpec = newSpecProvider.GetSpec(activation); + long? daoBlockNumber = newSpecProvider.DaoBlockNumber; - bool isMainnet = daoBlockNumber is not null; - if (isMainnet) - compareSpecsOptions |= CompareSpecsOptions.IsMainnet; + bool isMainnet = daoBlockNumber is not null; + if (isMainnet) + compareSpecsOptions |= CompareSpecsOptions.IsMainnet; - CompareSpecs(oldSpec, newSpec, activation, compareSpecsOptions); - } + CompareSpecs(oldSpec, newSpec, activation, compareSpecsOptions); } private static void CompareSpecs(IReleaseSpec expectedSpec, IReleaseSpec actualSpec, ForkActivation activation, CompareSpecsOptions compareSpecsOptions) @@ -828,6 +842,9 @@ public static IEnumerable BlockNumbersAndTimestampsNearForkActivations { get { + yield return new TestCaseData(new ForkActivation(1), true, false, false); + yield return new TestCaseData(new ForkActivation(2), true, false, false); + yield return new TestCaseData(new ForkActivation(3), true, false, false); yield return new TestCaseData(new ForkActivation(1, 9), true, false, false); yield return new TestCaseData(new ForkActivation(2, 9), true, false, false); yield return new TestCaseData(new ForkActivation(2, 10), true, true, false); diff --git a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecLoaderTests.cs b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecLoaderTests.cs index 0e2413c5898..d44ac4d2a4a 100644 --- a/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecLoaderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/ChainSpecStyle/ChainSpecLoaderTests.cs @@ -147,6 +147,7 @@ public void Can_load_goerli() chainSpec.LondonBlockNumber.Should().Be(GoerliSpecProvider.LondonBlockNumber); chainSpec.ShanghaiTimestamp.Should().Be(GoerliSpecProvider.ShanghaiTimestamp); chainSpec.ShanghaiTimestamp.Should().Be(GoerliSpecProvider.Instance.TimestampFork); + chainSpec.CancunTimestamp.Should().Be(GoerliSpecProvider.CancunTimestamp); } [Test] diff --git a/src/Nethermind/Nethermind.Specs.Test/CustomSpecProvider.cs b/src/Nethermind/Nethermind.Specs.Test/CustomSpecProvider.cs index 2a00eb2019d..6b405e37354 100644 --- a/src/Nethermind/Nethermind.Specs.Test/CustomSpecProvider.cs +++ b/src/Nethermind/Nethermind.Specs.Test/CustomSpecProvider.cs @@ -1,86 +1,59 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using System.Linq; using Nethermind.Core; -using Nethermind.Core.Collections; using Nethermind.Core.Specs; using Nethermind.Int256; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Specs.Forks; -namespace Nethermind.Specs.Test -{ - public class CustomSpecProvider : ISpecProvider - { - private ForkActivation? _theMergeBlock = null; - private readonly (ForkActivation Activation, IReleaseSpec Spec)[] _transitions; - - public ulong NetworkId { get; } - public ulong ChainId { get; } +namespace Nethermind.Specs.Test; - public ForkActivation[] TransitionActivations { get; } - - public CustomSpecProvider(params (ForkActivation Activation, IReleaseSpec Spec)[] transitions) : this(TestBlockchainIds.NetworkId, TestBlockchainIds.ChainId, transitions) - { - } - - public CustomSpecProvider(ulong networkId, ulong chainId, params (ForkActivation Activation, IReleaseSpec Spec)[] transitions) - { - NetworkId = networkId; - ChainId = chainId; +public class CustomSpecProvider : SpecProviderBase, ISpecProvider +{ + private ForkActivation? _theMergeBlock = null; - if (transitions.Length == 0) - { - throw new ArgumentException($"There must be at least one release specified when instantiating {nameof(CustomSpecProvider)}", $"{nameof(transitions)}"); - } + public ulong NetworkId { get; } + public ulong ChainId { get; } - _transitions = transitions.OrderBy(r => r.Activation).ToArray(); - TransitionActivations = _transitions.Select(t => t.Activation).ToArray(); + public CustomSpecProvider(params (ForkActivation Activation, IReleaseSpec Spec)[] transitions) : this(TestBlockchainIds.NetworkId, TestBlockchainIds.ChainId, transitions) + { + } - if (transitions[0].Activation.BlockNumber != 0L) - { - throw new ArgumentException($"First release specified when instantiating {nameof(CustomSpecProvider)} should be at genesis block (0)", $"{nameof(transitions)}"); - } - } + public CustomSpecProvider(ulong networkId, ulong chainId, params (ForkActivation Activation, IReleaseSpec Spec)[] transitions) + { + NetworkId = networkId; + ChainId = chainId; - public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalDifficulty = null) - { - if (blockNumber is not null) - _theMergeBlock = (ForkActivation)blockNumber; - if (terminalTotalDifficulty is not null) - TerminalTotalDifficulty = terminalTotalDifficulty; - } + (ForkActivation Activation, IReleaseSpec Spec)[] orderedTransitions = transitions.OrderBy(r => r.Activation).ToArray(); - public ForkActivation? MergeBlockNumber => _theMergeBlock; + LoadTransitions(orderedTransitions); - public ulong TimestampFork { get; set; } = ISpecProvider.TimestampForkNever; - public UInt256? TerminalTotalDifficulty { get; set; } + TransitionActivations = orderedTransitions.Select(t => t.Activation).ToArray(); + } -#pragma warning disable CS8602 -#pragma warning disable CS8603 - public IReleaseSpec GenesisSpec => _transitions.Length == 0 ? null : _transitions[0].Spec; -#pragma warning restore CS8603 -#pragma warning restore CS8602 + public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalDifficulty = null) + { + if (blockNumber is not null) + _theMergeBlock = (ForkActivation)blockNumber; + if (terminalTotalDifficulty is not null) + TerminalTotalDifficulty = terminalTotalDifficulty; + } - public IReleaseSpec GetSpec(ForkActivation forkActivation) => - _transitions.TryGetSearchedItem(forkActivation, - CompareTransitionOnBlock, - out (ForkActivation Activation, IReleaseSpec Spec) transition) - ? transition.Spec - : GenesisSpec; + public ForkActivation? MergeBlockNumber => _theMergeBlock; - private static int CompareTransitionOnBlock(ForkActivation forkActivation, (ForkActivation Activation, IReleaseSpec Spec) transition) => - forkActivation.CompareTo(transition.Activation); + public ulong TimestampFork { get; set; } = ISpecProvider.TimestampForkNever; + public UInt256? TerminalTotalDifficulty { get; set; } - public long? DaoBlockNumber + public long? DaoBlockNumber + { + get { - get - { - (ForkActivation forkActivation, IReleaseSpec daoRelease) = _transitions.SingleOrDefault(t => t.Spec == Dao.Instance); - return daoRelease is not null ? forkActivation.BlockNumber : null; - } + (ForkActivation forkActivation, IReleaseSpec? daoRelease) = _blockTransitions.SingleOrDefault(t => t.Spec == Dao.Instance); + return daoRelease is not null ? forkActivation.BlockNumber : null; } - } + } + diff --git a/src/Nethermind/Nethermind.Specs.Test/CustomSpecProviderTests.cs b/src/Nethermind/Nethermind.Specs.Test/CustomSpecProviderTests.cs index 76ecf40eb11..df91b86c5e5 100644 --- a/src/Nethermind/Nethermind.Specs.Test/CustomSpecProviderTests.cs +++ b/src/Nethermind/Nethermind.Specs.Test/CustomSpecProviderTests.cs @@ -21,10 +21,6 @@ public void When_no_transitions_specified_throws_argument_exception() public void When_first_release_is_not_at_block_zero_then_throws_argument_exception() { Assert.Throws(() => _ = new CustomSpecProvider(((ForkActivation)1, Byzantium.Instance)), "ordered"); - - Assert.Throws(() => _ = new CustomSpecProvider( - ((ForkActivation)1, Byzantium.Instance), - ((ForkActivation)0, Frontier.Instance)), "not ordered"); } [Test] diff --git a/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs b/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs index 225168c5b1d..b36a57071d0 100644 --- a/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs.Test/OverridableReleaseSpec.cs @@ -157,5 +157,8 @@ public ulong Eip4844TransitionTimestamp public bool IsEip6780Enabled => _spec.IsEip6780Enabled; public bool IsEip4788Enabled => _spec.IsEip4788Enabled; public Address Eip4788ContractAddress => _spec.Eip4788ContractAddress; + public UInt256 ForkBaseFee => _spec.ForkBaseFee; + public UInt256 BaseFeeMaxChangeDenominator => _spec.BaseFeeMaxChangeDenominator; + public long ElasticityMultiplier => _spec.ElasticityMultiplier; } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs index 2b5f645602b..817250bc01e 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainParameters.cs @@ -54,11 +54,11 @@ public class ChainParameters public long? Eip3541Transition { get; set; } public long? Eip3607Transition { get; set; } - public UInt256 Eip1559BaseFeeInitialValue { get; set; } + public UInt256? Eip1559BaseFeeInitialValue { get; set; } - public UInt256 Eip1559BaseFeeMaxChangeDenominator { get; set; } + public UInt256? Eip1559BaseFeeMaxChangeDenominator { get; set; } - public long Eip1559ElasticityMultiplier { get; set; } + public long? Eip1559ElasticityMultiplier { get; set; } /// /// Transaction permission managing contract address. diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs index 64d5257070b..7aad4bac04f 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecBasedSpecProvider.cs @@ -6,26 +6,21 @@ using System.Linq; using System.Numerics; using System.Reflection; -using Nethermind.Core.Collections; +using Nethermind.Core; using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.Logging; namespace Nethermind.Specs.ChainSpecStyle { - public class ChainSpecBasedSpecProvider : ISpecProvider + public class ChainSpecBasedSpecProvider : SpecProviderBase, ISpecProvider { - private (ForkActivation Activation, ReleaseSpec Spec)[] _transitions; - private (ForkActivation Activation, ReleaseSpec Spec)[] _timestampOnlyTransitions; - private ForkActivation? _firstTimestampActivation; - private readonly ChainSpec _chainSpec; - private readonly ILogger _logger; public ChainSpecBasedSpecProvider(ChainSpec chainSpec, ILogManager logManager = null) + : base(logManager?.GetClassLogger() ?? LimboTraceLogger.Instance) { _chainSpec = chainSpec ?? throw new ArgumentNullException(nameof(chainSpec)); - _logger = logManager?.GetClassLogger() ?? LimboTraceLogger.Instance; BuildTransitions(); } @@ -96,10 +91,12 @@ static void Add(SortedSet transitions, T value, T? minValueExclusive) transitionBlockNumbers.Add(bombDelay.Key); } + + (ForkActivation Activation, IReleaseSpec Spec)[] allTransitions = CreateTransitions(_chainSpec, transitionBlockNumbers, transitionTimestamps); + + LoadTransitions(allTransitions); + TransitionActivations = CreateTransitionActivations(transitionBlockNumbers, transitionTimestamps); - _transitions = CreateTransitions(_chainSpec, transitionBlockNumbers, transitionTimestamps); - _firstTimestampActivation = TransitionActivations.FirstOrDefault(t => t.Timestamp is not null); - _timestampOnlyTransitions = _transitions.SkipWhile(t => t.Activation.Timestamp is null).ToArray(); if (_chainSpec.Parameters.TerminalPoWBlockNumber is not null) { @@ -109,12 +106,12 @@ static void Add(SortedSet transitions, T value, T? minValueExclusive) TerminalTotalDifficulty = _chainSpec.Parameters.TerminalTotalDifficulty; } - private static (ForkActivation, ReleaseSpec Spec)[] CreateTransitions( + private static (ForkActivation, IReleaseSpec Spec)[] CreateTransitions( ChainSpec chainSpec, SortedSet transitionBlockNumbers, SortedSet transitionTimestamps) { - (ForkActivation Activation, ReleaseSpec Spec)[] transitions = new (ForkActivation, ReleaseSpec Spec)[transitionBlockNumbers.Count + transitionTimestamps.Count]; + (ForkActivation Activation, IReleaseSpec Spec)[] transitions = new (ForkActivation, IReleaseSpec Spec)[transitionBlockNumbers.Count + transitionTimestamps.Count]; long biggestBlockTransition = transitionBlockNumbers.Max; int index = 0; @@ -209,8 +206,18 @@ private static ReleaseSpec CreateReleaseSpec(ChainSpec chainSpec, long releaseSt releaseSpec.IsEip3607Enabled = (chainSpec.Parameters.Eip3607Transition ?? long.MaxValue) <= releaseStartBlock; releaseSpec.ValidateChainId = (chainSpec.Parameters.ValidateChainIdTransition ?? 0) <= releaseStartBlock; releaseSpec.ValidateReceipts = ((chainSpec.Parameters.ValidateReceiptsTransition > 0) ? Math.Max(chainSpec.Parameters.ValidateReceiptsTransition ?? 0, chainSpec.Parameters.Eip658Transition ?? 0) : 0) <= releaseStartBlock; + releaseSpec.Eip1559FeeCollector = releaseSpec.IsEip1559Enabled && (chainSpec.Parameters.Eip1559FeeCollectorTransition ?? long.MaxValue) <= releaseStartBlock ? chainSpec.Parameters.Eip1559FeeCollector : null; releaseSpec.Eip1559BaseFeeMinValue = releaseSpec.IsEip1559Enabled && (chainSpec.Parameters.Eip1559BaseFeeMinValueTransition ?? long.MaxValue) <= releaseStartBlock ? chainSpec.Parameters.Eip1559BaseFeeMinValue : null; + releaseSpec.ElasticityMultiplier = chainSpec.Parameters.Eip1559ElasticityMultiplier ?? Eip1559Constants.DefaultElasticityMultiplier; + releaseSpec.ForkBaseFee = chainSpec.Parameters.Eip1559BaseFeeInitialValue ?? Eip1559Constants.DefaultForkBaseFee; + releaseSpec.BaseFeeMaxChangeDenominator = chainSpec.Parameters.Eip1559BaseFeeMaxChangeDenominator ?? Eip1559Constants.DefaultBaseFeeMaxChangeDenominator; + + if (chainSpec.Optimism?.CanyonTimestamp <= releaseStartTimestamp) + { + releaseSpec.BaseFeeMaxChangeDenominator = chainSpec.Optimism.CanyonBaseFeeChangeDenominator; + } + if (chainSpec.Ethash is not null) { @@ -266,41 +273,9 @@ public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalD public UInt256? TerminalTotalDifficulty { get; private set; } - public IReleaseSpec GenesisSpec => _transitions.Length == 0 ? null : _transitions[0].Spec; - - public IReleaseSpec GetSpec(ForkActivation activation) - { - (ForkActivation Activation, ReleaseSpec Spec)[] consideredTransitions = _transitions; - - // TODO: Is this actually needed? Can this be tricked with invalid activation check if someone would fake timestamp from the future? - if (_firstTimestampActivation is not null && activation.Timestamp is not null) - { - if (_firstTimestampActivation.Value.Timestamp < activation.Timestamp - && _firstTimestampActivation.Value.BlockNumber > activation.BlockNumber) - { - if (_logger.IsWarn) _logger.Warn($"Chainspec file is misconfigured! Timestamp transition is configured to happen before the last block transition."); - } - - if (_firstTimestampActivation.Value.Timestamp <= activation.Timestamp) - { - consideredTransitions = _timestampOnlyTransitions; - } - } - - return consideredTransitions.TryGetSearchedItem(activation, - CompareTransitionOnActivation, - out (ForkActivation Activation, ReleaseSpec Spec) transition) - ? transition.Spec - : GenesisSpec; - } - - private static int CompareTransitionOnActivation(ForkActivation activation, (ForkActivation Activation, ReleaseSpec Spec) transition) => - activation.CompareTo(transition.Activation); - public long? DaoBlockNumber => _chainSpec.DaoForkBlockNumber; public ulong NetworkId => _chainSpec.NetworkId; public ulong ChainId => _chainSpec.ChainId; - public ForkActivation[] TransitionActivations { get; private set; } } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs index 163834c1a40..ec649bbda53 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/ChainSpecLoader.cs @@ -157,10 +157,10 @@ bool GetForInnerPathExistence(KeyValuePair o) TransactionPermissionContractTransition = chainSpecJson.Params.TransactionPermissionContractTransition, ValidateChainIdTransition = chainSpecJson.Params.ValidateChainIdTransition, ValidateReceiptsTransition = chainSpecJson.Params.ValidateReceiptsTransition, - Eip1559ElasticityMultiplier = chainSpecJson.Params.Eip1559ElasticityMultiplier ?? Eip1559Constants.ElasticityMultiplier, - Eip1559BaseFeeInitialValue = chainSpecJson.Params.Eip1559BaseFeeInitialValue ?? Eip1559Constants.ForkBaseFee, + Eip1559ElasticityMultiplier = chainSpecJson.Params.Eip1559ElasticityMultiplier ?? Eip1559Constants.DefaultElasticityMultiplier, + Eip1559BaseFeeInitialValue = chainSpecJson.Params.Eip1559BaseFeeInitialValue ?? Eip1559Constants.DefaultForkBaseFee, Eip1559BaseFeeMaxChangeDenominator = chainSpecJson.Params.Eip1559BaseFeeMaxChangeDenominator ?? - Eip1559Constants.BaseFeeMaxChangeDenominator, + Eip1559Constants.DefaultBaseFeeMaxChangeDenominator, Eip1559FeeCollector = chainSpecJson.Params.Eip1559FeeCollector, Eip1559FeeCollectorTransition = chainSpecJson.Params.Eip1559FeeCollectorTransition, Eip1559BaseFeeMinValueTransition = chainSpecJson.Params.Eip1559BaseFeeMinValueTransition, @@ -180,10 +180,6 @@ bool GetForInnerPathExistence(KeyValuePair o) ?? GetTransitionForExpectedPricing("alt_bn128_pairing", "price.alt_bn128_pairing.base", 45000); chainSpec.Parameters.Eip2565Transition ??= GetTransitionIfInnerPathExists("modexp", "price.modexp2565"); - Eip1559Constants.ElasticityMultiplier = chainSpec.Parameters.Eip1559ElasticityMultiplier; - Eip1559Constants.ForkBaseFee = chainSpec.Parameters.Eip1559BaseFeeInitialValue; - Eip1559Constants.BaseFeeMaxChangeDenominator = chainSpec.Parameters.Eip1559BaseFeeMaxChangeDenominator; - Eip4844Constants.OverrideIfAny( chainSpec.Parameters.Eip4844BlobGasPriceUpdateFraction, chainSpec.Parameters.Eip4844MaxBlobGasPerBlock, @@ -347,8 +343,12 @@ static AuRaParameters.Validator LoadValidator(ChainSpecJson.AuRaValidatorJson va { RegolithTimestamp = chainSpecJson.Engine.Optimism.RegolithTimestamp, BedrockBlockNumber = chainSpecJson.Engine.Optimism.BedrockBlockNumber, + CanyonTimestamp = chainSpecJson.Engine.Optimism.CanyonTimestamp, L1FeeRecipient = chainSpecJson.Engine.Optimism.L1FeeRecipient, - L1BlockAddress = chainSpecJson.Engine.Optimism.L1BlockAddress + L1BlockAddress = chainSpecJson.Engine.Optimism.L1BlockAddress, + CanyonBaseFeeChangeDenominator = chainSpecJson.Engine.Optimism.CanyonBaseFeeChangeDenominator, + Create2DeployerAddress = chainSpecJson.Engine.Optimism.Create2DeployerAddress, + Create2DeployerCode = chainSpecJson.Engine.Optimism.Create2DeployerCode }; } else if (chainSpecJson.Engine?.NethDev is not null) diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecGenesisJson.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecGenesisJson.cs index c04038cafe0..a72fb549b86 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecGenesisJson.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecGenesisJson.cs @@ -24,8 +24,8 @@ internal class ChainSpecGenesisJson public bool StateUnavailable { get; set; } = false; public Hash256 StateRoot { get; set; } - public ulong? BlobGasUsed { get; set; } - public ulong? ExcessBlobGas { get; set; } + public ulong BlobGasUsed { get; set; } + public ulong ExcessBlobGas { get; set; } public Hash256 ParentBeaconBlockRoot { get; set; } } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs index 4bf9b38fe2f..d819732c34f 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/Json/ChainSpecJson.cs @@ -7,6 +7,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Nethermind.Core; +using Nethermind.Core.Crypto; using Nethermind.Int256; namespace Nethermind.Specs.ChainSpecStyle.Json @@ -171,8 +172,12 @@ internal class OptimismEngineJson { public ulong RegolithTimestamp => Params.RegolithTimestamp; public long BedrockBlockNumber => Params.BedrockBlockNumber; + public ulong? CanyonTimestamp => Params.CanyonTimestamp; public Address L1FeeRecipient => Params.L1FeeRecipient; public Address L1BlockAddress => Params.L1BlockAddress; + public UInt256 CanyonBaseFeeChangeDenominator => Params.CanyonBaseFeeChangeDenominator; + public Address Create2DeployerAddress => Params.Create2DeployerAddress; + public byte[] Create2DeployerCode => Params.Create2DeployerCode; public OptimismEngineParamsJson Params { get; set; } } @@ -180,8 +185,12 @@ internal class OptimismEngineParamsJson { public ulong RegolithTimestamp { get; set; } public long BedrockBlockNumber { get; set; } + public ulong? CanyonTimestamp { get; set; } public Address L1FeeRecipient { get; set; } public Address L1BlockAddress { get; set; } + public UInt256 CanyonBaseFeeChangeDenominator { get; set; } + public Address Create2DeployerAddress { get; set; } + public byte[] Create2DeployerCode { get; set; } } internal class NethDevJson diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs index 586481dbc96..b2439224257 100644 --- a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/OptimismParameters.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Collections.Generic; using Nethermind.Core; using Nethermind.Int256; @@ -13,8 +12,16 @@ public class OptimismParameters public long BedrockBlockNumber { get; set; } + public ulong? CanyonTimestamp { get; set; } + public Address L1FeeRecipient { get; set; } public Address L1BlockAddress { get; set; } + + public UInt256 CanyonBaseFeeChangeDenominator { get; set; } + + public Address Create2DeployerAddress { get; set; } + + public byte[] Create2DeployerCode { get; set; } } } diff --git a/src/Nethermind/Nethermind.Specs/ChainSpecStyle/SpecProviderBase.cs b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/SpecProviderBase.cs new file mode 100644 index 00000000000..bc5410bb807 --- /dev/null +++ b/src/Nethermind/Nethermind.Specs/ChainSpecStyle/SpecProviderBase.cs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Collections; +using Nethermind.Core.Specs; +using Nethermind.Logging; +using System; +using System.Linq; + +namespace Nethermind.Specs.ChainSpecStyle; + +public abstract class SpecProviderBase +{ + protected (ForkActivation Activation, IReleaseSpec Spec)[] _blockTransitions; + private (ForkActivation Activation, IReleaseSpec Spec)[] _timestampTransitions; + private ForkActivation? _firstTimestampActivation; + protected readonly ILogger _logger; + + public SpecProviderBase(ILogger logger = null) + { + _logger = logger; + } + + protected void LoadTransitions((ForkActivation Activation, IReleaseSpec Spec)[] transitions) + { + if (transitions.Length == 0) + { + throw new ArgumentException($"There must be at least one release specified when instantiating {GetType()}", $"{nameof(transitions)}"); + } + + if (transitions.First().Activation.BlockNumber != 0L) + { + throw new ArgumentException($"First release specified when instantiating {GetType()} should be at genesis block (0)", $"{nameof(transitions)}"); + } + + _blockTransitions = transitions.TakeWhile(t => t.Activation.Timestamp is null).ToArray(); + _timestampTransitions = transitions.SkipWhile(t => t.Activation.Timestamp is null).ToArray(); + _firstTimestampActivation = _timestampTransitions.Length != 0 ? _timestampTransitions.First().Activation : null; + GenesisSpec = transitions.First().Spec; + } + + public ForkActivation[] TransitionActivations { get; protected set; } + + public IReleaseSpec GenesisSpec { get; private set; } + + public IReleaseSpec GetSpec(ForkActivation activation) + { + static int CompareTransitionOnActivation(ForkActivation activation, (ForkActivation Activation, IReleaseSpec Spec) transition) => + activation.CompareTo(transition.Activation); + + (ForkActivation Activation, IReleaseSpec Spec)[] consideredTransitions = _blockTransitions; + + if (_firstTimestampActivation is not null && activation.Timestamp is not null) + { + if (_firstTimestampActivation.Value.Timestamp < activation.Timestamp + && _firstTimestampActivation.Value.BlockNumber > activation.BlockNumber) + { + if (_logger is not null && _logger.IsWarn) _logger.Warn($"Chainspec file is misconfigured! Timestamp transition is configured to happen before the last block transition."); + } + + if (_firstTimestampActivation.Value.Timestamp <= activation.Timestamp) + consideredTransitions = _timestampTransitions; + } + + return consideredTransitions.TryGetSearchedItem(activation, + CompareTransitionOnActivation, + out (ForkActivation Activation, IReleaseSpec Spec) transition) + ? transition.Spec + : GenesisSpec; + } +} diff --git a/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs b/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs index b92ae9e61a4..a6d72720ba2 100644 --- a/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs +++ b/src/Nethermind/Nethermind.Specs/Forks/16_Cancun.cs @@ -15,10 +15,10 @@ protected Cancun() { Name = "Cancun"; IsEip1153Enabled = true; - IsEip5656Enabled = true; + IsEip4788Enabled = true; IsEip4844Enabled = true; + IsEip5656Enabled = true; IsEip6780Enabled = true; - IsEip4788Enabled = true; Eip4788ContractAddress = new Address("0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02"); } diff --git a/src/Nethermind/Nethermind.Specs/GoerliSpecProvider.cs b/src/Nethermind/Nethermind.Specs/GoerliSpecProvider.cs index 5defde805f0..13b68ffbe88 100644 --- a/src/Nethermind/Nethermind.Specs/GoerliSpecProvider.cs +++ b/src/Nethermind/Nethermind.Specs/GoerliSpecProvider.cs @@ -15,6 +15,7 @@ public class GoerliSpecProvider : ISpecProvider public const long LondonBlockNumber = 5_062_605; public const ulong BeaconChainGenesisTimestamp = 0x6059f460; public const ulong ShanghaiTimestamp = 0x6410f460; + public const ulong CancunTimestamp = 0x65A77460; private GoerliSpecProvider() { } @@ -28,7 +29,8 @@ public IReleaseSpec GetSpec(ForkActivation forkActivation) _ => forkActivation.Timestamp switch { null or < ShanghaiTimestamp => London.Instance, - _ => Shanghai.Instance + < CancunTimestamp => Shanghai.Instance, + _ => Cancun.Instance } }; } @@ -53,7 +55,8 @@ public void UpdateMergeTransitionInfo(long? blockNumber, UInt256? terminalTotalD (ForkActivation)IstanbulBlockNumber, (ForkActivation)BerlinBlockNumber, (ForkActivation)LondonBlockNumber, - (LondonBlockNumber, ShanghaiTimestamp) + (LondonBlockNumber + 1, ShanghaiTimestamp), + (LondonBlockNumber + 2, CancunTimestamp) }; public static readonly GoerliSpecProvider Instance = new(); diff --git a/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs b/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs index fa886f82fa6..5de83c8356b 100644 --- a/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs/ReleaseSpec.cs @@ -72,6 +72,9 @@ public ReleaseSpec Clone() public ulong Eip4844TransitionTimestamp { get; set; } public Address Eip1559FeeCollector { get; set; } public UInt256? Eip1559BaseFeeMinValue { get; set; } + public UInt256 ForkBaseFee { get; set; } = Eip1559Constants.DefaultForkBaseFee; + public UInt256 BaseFeeMaxChangeDenominator { get; set; } = Eip1559Constants.DefaultBaseFeeMaxChangeDenominator; + public long ElasticityMultiplier { get; set; } = Eip1559Constants.DefaultElasticityMultiplier; public bool IsEip1153Enabled { get; set; } public bool IsEip3651Enabled { get; set; } public bool IsEip3855Enabled { get; set; } diff --git a/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs b/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs index b4c0c4d29e8..8774d7a03ae 100644 --- a/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs +++ b/src/Nethermind/Nethermind.Specs/SystemTransactionReleaseSpec.cs @@ -130,5 +130,8 @@ public bool IsEip158IgnoredAccount(Address address) public bool IsEip6780Enabled => _spec.IsEip6780Enabled; public bool IsEip4788Enabled => _spec.IsEip4788Enabled; public Address Eip4788ContractAddress => _spec.Eip4788ContractAddress; + public UInt256 ForkBaseFee => _spec.ForkBaseFee; + public UInt256 BaseFeeMaxChangeDenominator => _spec.BaseFeeMaxChangeDenominator; + public long ElasticityMultiplier => _spec.ElasticityMultiplier; } } diff --git a/src/Nethermind/Nethermind.State/StateProvider.cs b/src/Nethermind/Nethermind.State/StateProvider.cs index 1d0cfa2ac28..4ecaff55d2a 100644 --- a/src/Nethermind/Nethermind.State/StateProvider.cs +++ b/src/Nethermind/Nethermind.State/StateProvider.cs @@ -26,6 +26,7 @@ internal class StateProvider private const int StartCapacity = Resettable.StartCapacity; private readonly ResettableDictionary> _intraBlockCache = new(); private readonly ResettableHashSet
_committedThisRound = new(); + private readonly HashSet
_nullAccountReads = new(); // Only guarding against hot duplicates so filter doesn't need to be too big // Note: // False negatives are fine as they will just result in a overwrite set @@ -508,7 +509,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool // because it was not committed yet it means that the just cache is the only state (so it was read only) if (isTracing && change.ChangeType == ChangeType.JustCache) { - _readsForTracing.Add(change.Address); + _nullAccountReads.Add(change.Address); continue; } @@ -597,7 +598,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool if (isTracing) { - foreach (Address nullRead in _readsForTracing) + foreach (Address nullRead in _nullAccountReads) { // // this may be enough, let us write tests stateTracer.ReportAccountRead(nullRead); @@ -606,7 +607,7 @@ public void Commit(IReleaseSpec releaseSpec, IWorldStateTracer stateTracer, bool Resettable.Reset(ref _changes, ref _capacity, ref _currentPosition, StartCapacity); _committedThisRound.Reset(); - _readsForTracing.Clear(); + _nullAccountReads.Clear(); _intraBlockCache.Reset(); if (isTracing) @@ -687,10 +688,10 @@ private void SetState(Address address, Account? account) _tree.Set(address, account); } - private readonly HashSet
_readsForTracing = new(); - private Account? GetAndAddToCache(Address address) { + if (_nullAccountReads.Contains(address)) return null; + Account? account = GetState(address); if (account is not null) { @@ -699,7 +700,7 @@ private void SetState(Address address, Account? account) else { // just for tracing - potential perf hit, maybe a better solution? - _readsForTracing.Add(address); + _nullAccountReads.Add(address); } return account; @@ -803,7 +804,7 @@ public void Reset() if (_logger.IsTrace) _logger.Trace("Clearing state provider caches"); _intraBlockCache.Reset(); _committedThisRound.Reset(); - _readsForTracing.Clear(); + _nullAccountReads.Clear(); _currentPosition = Resettable.EmptyPosition; Array.Clear(_changes, 0, _changes.Length); _needsStateRootUpdate = false; diff --git a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs index 2800d2910bd..10ae29c0c92 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/BlockDownloaderTests.Merge.cs @@ -20,6 +20,7 @@ using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Merge.Plugin.Test; using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Stats; using Nethermind.Stats.Model; using Nethermind.Synchronization.Blocks; @@ -456,6 +457,7 @@ public MergeConfig MergeConfig MetadataDb, BlockTree, SpecProvider, + new ChainSpec(), LimboLogs.Instance); protected override IBetterPeerStrategy BetterPeerStrategy => _betterPeerStrategy ??= diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs index a44fdcf7342..49a30c32adc 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/StateSyncFeedTests.cs @@ -28,7 +28,7 @@ namespace Nethermind.Synchronization.Test.FastSync [TestFixture(1, 100)] [TestFixture(4, 0)] [TestFixture(4, 100)] - [Parallelizable(ParallelScope.Children)] + [Parallelizable(ParallelScope.All)] public class StateSyncFeedTests : StateSyncFeedTestsBase { // Useful for set and forget run. But this test is taking a long time to have it set to other than 1. diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs index f702e409258..921b5baa7df 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SnapSync/ProgressTrackerTests.cs @@ -6,6 +6,7 @@ using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Core.Crypto; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; using Nethermind.Db; using Nethermind.Logging; @@ -135,4 +136,24 @@ public void Will_deque_storage_request_if_high() request.CodesRequest.Should().BeNull(); request.StorageRangeRequest.Should().NotBeNull(); } + + [Test] + public void Will_mark_progress_and_flush_when_finished() + { + BlockTree blockTree = Build.A.BlockTree().WithBlocks(Build.A.Block + .WithStateRoot(Keccak.EmptyTreeHash) + .TestObject).TestObject; + TestMemDb memDb = new TestMemDb(); + ProgressTracker progressTracker = new ProgressTracker(blockTree, memDb, LimboLogs.Instance, 1); + + (SnapSyncBatch request, bool finished) = progressTracker.GetNextRequest(); + request.AccountRangeRequest.Should().NotBeNull(); + progressTracker.UpdateAccountRangePartitionProgress(request.AccountRangeRequest!.LimitHash!.Value, Keccak.MaxValue, false); + progressTracker.ReportAccountRangePartitionFinished(request.AccountRangeRequest!.LimitHash!.Value); + (_, finished) = progressTracker.GetNextRequest(); + finished.Should().BeTrue(); + + memDb.WasFlushed.Should().BeTrue(); + memDb[ProgressTracker.ACC_PROGRESS_KEY].Should().BeEquivalentTo(Keccak.MaxValue.BytesToArray()); + } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs index 5394cd8e754..8ecab27bc76 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncServerTests.cs @@ -22,6 +22,7 @@ using Nethermind.Merge.Plugin.Handlers; using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Specs; +using Nethermind.Specs.ChainSpecStyle; using Nethermind.Specs.Forks; using Nethermind.State; using Nethermind.Stats.Model; @@ -180,6 +181,7 @@ public void Terminal_block_with_lower_td_should_not_change_best_suggested_but_sh new MemDb(), localBlockTree, testSpecProvider, + new ChainSpec(), LimboLogs.Instance); HeaderValidator headerValidator = new( localBlockTree, @@ -389,6 +391,7 @@ private Context CreateMergeContext(int blockTreeChainLength, UInt256 ttd) new MemDb(), localBlockTree, testSpecProvider, + new ChainSpec(), LimboLogs.Instance); MergeSealEngine sealEngine = new( new SealEngine(new NethDevSealEngine(), Always.Valid), diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs index a1ed58c7617..3d2cfd28eef 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SynchronizerTests.cs @@ -324,7 +324,7 @@ ISyncConfig GetSyncConfig() => { mergeConfig.TerminalTotalDifficulty = UInt256.MaxValue.ToString(CultureInfo.InvariantCulture); } - PoSSwitcher poSSwitcher = new(mergeConfig, syncConfig, dbProvider.MetadataDb, BlockTree, new TestSingleReleaseSpecProvider(Constantinople.Instance), _logManager); + PoSSwitcher poSSwitcher = new(mergeConfig, syncConfig, dbProvider.MetadataDb, BlockTree, new TestSingleReleaseSpecProvider(Constantinople.Instance), new ChainSpec(), _logManager); IBeaconPivot beaconPivot = new BeaconPivot(syncConfig, dbProvider.MetadataDb, BlockTree, _logManager); TrieStore trieStore = new(stateDb, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs index febb468f2ad..72cdb8a94c6 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/DetailedProgress.cs @@ -79,7 +79,7 @@ internal void DisplayProgressReport(int pendingRequestsCount, BranchProgress bra } if (logger.IsInfo) logger.Info( - $"State Sync {TimeSpan.FromSeconds(SecondsInSync):dd\\.hh\\:mm\\:ss} | {dataSizeInfo} | branches: {branchProgress.Progress:P2} | kB/s: {savedKBytesPerSecond,5:F0} | accounts {SavedAccounts} | nodes {SavedNodesCount} | diagnostics: {pendingRequestsCount}.{AverageTimeInHandler:f2}ms"); + $"State Sync {TimeSpan.FromSeconds(SecondsInSync):dd\\.hh\\:mm\\:ss} | {dataSizeInfo} | branches: {branchProgress.Progress:P2} | kB/s: {savedKBytesPerSecond,5:F0} | accounts {SavedAccounts} | nodes {SavedNodesCount} | pending: {pendingRequestsCount,3} | ave: {AverageTimeInHandler:f2}ms"); if (logger.IsDebug && DateTime.UtcNow - LastReportTime.full > TimeSpan.FromSeconds(10)) { long allChecks = CheckWasInDependencies + CheckWasCached + StateWasThere + StateWasNotThere; diff --git a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs index ba6f6f482c3..61c3821d8a9 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastSync/TreeSync.cs @@ -308,6 +308,7 @@ shorter than the request */ _data.DisplayProgressReport(_pendingRequests.Count, _branchProgress, _logger); + handleWatch.Stop(); long total = handleWatch.ElapsedMilliseconds + _networkWatch.ElapsedMilliseconds; if (total != 0) { @@ -326,9 +327,7 @@ shorter than the request */ Interlocked.Add(ref _handleWatch, handleWatch.ElapsedMilliseconds); _data.LastDbReads = _data.DbChecks; - _data.AverageTimeInHandler = - (_data.AverageTimeInHandler * (_data.ProcessedRequestsCount - 1) + - _handleWatch) / _data.ProcessedRequestsCount; + _data.AverageTimeInHandler = _handleWatch / (decimal)_data.ProcessedRequestsCount; Interlocked.Add(ref _data.HandledNodesCount, nonEmptyResponses); return result; diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs index 15b99c6c7f8..f73edf4c79c 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/ProgressTracker.cs @@ -24,7 +24,7 @@ public class ProgressTracker public const int HIGH_STORAGE_QUEUE_SIZE = STORAGE_BATCH_SIZE * 100; private const int CODES_BATCH_SIZE = 1_000; public const int HIGH_CODES_QUEUE_SIZE = CODES_BATCH_SIZE * 5; - private readonly byte[] ACC_PROGRESS_KEY = Encoding.ASCII.GetBytes("AccountProgressKey"); + internal static readonly byte[] ACC_PROGRESS_KEY = Encoding.ASCII.GetBytes("AccountProgressKey"); // This does not need to be a lot as it spawn other requests. In fact 8 is probably too much. It is severely // bottlenecked by _syncCommit lock in SnapProviderHelper, which in turns is limited by the IO. @@ -416,7 +416,8 @@ private void GetSyncProgress() private void FinishRangePhase() { - _db.PutSpan(ACC_PROGRESS_KEY, ValueKeccak.MaxValue.Bytes); + _db.PutSpan(ACC_PROGRESS_KEY, ValueKeccak.MaxValue.Bytes, WriteFlags.DisableWAL); + _db.Flush(); } private void LogRequest(string reqType) diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 98d3a0cebf6..d2f51406c0e 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -273,7 +273,7 @@ public void AddCodes(ValueHash256[] requestedHashes, byte[][] codes) { HashSet set = requestedHashes.ToHashSet(); - using (IWriteBatch writeWriteBatch = _dbProvider.CodeDb.StartWriteBatch()) + using (IWriteBatch writeBatch = _dbProvider.CodeDb.StartWriteBatch()) { for (int i = 0; i < codes.Length; i++) { @@ -283,7 +283,7 @@ public void AddCodes(ValueHash256[] requestedHashes, byte[][] codes) if (set.Remove(codeHash)) { Interlocked.Add(ref Metrics.SnapStateSynced, code.Length); - writeWriteBatch[codeHash.Bytes] = code; + writeBatch[codeHash.Bytes] = code; } } } diff --git a/src/Nethermind/Nethermind.Trie/ITreeVisitor.cs b/src/Nethermind/Nethermind.Trie/ITreeVisitor.cs index 6cdd3ecd553..48579ca1804 100644 --- a/src/Nethermind/Nethermind.Trie/ITreeVisitor.cs +++ b/src/Nethermind/Nethermind.Trie/ITreeVisitor.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using Nethermind.Core; using Nethermind.Core.Crypto; namespace Nethermind.Trie @@ -12,6 +13,8 @@ public interface ITreeVisitor ///
public bool IsFullDbScan { get; } + ReadFlags ExtraReadFlag => ReadFlags.None; + bool ShouldVisit(Hash256 nextNode); void VisitTree(Hash256 rootHash, TrieVisitContext trieVisitContext); diff --git a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs index cdcc6361753..1cedadc144e 100644 --- a/src/Nethermind/Nethermind.Trie/PatriciaTree.cs +++ b/src/Nethermind/Nethermind.Trie/PatriciaTree.cs @@ -53,6 +53,9 @@ public class PatriciaTree private readonly bool _allowCommits; + private readonly bool _isTrace; + private readonly bool _isDebug; + private Hash256 _rootHash = Keccak.EmptyTreeHash; public TrieNode? RootRef { get; set; } @@ -116,6 +119,8 @@ public PatriciaTree( ICappedArrayPool? bufferPool = null) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _isTrace = _logger.IsTrace; + _isDebug = _logger.IsDebug; TrieStore = trieStore ?? throw new ArgumentNullException(nameof(trieStore)); _parallelBranches = parallelBranches; _allowCommits = allowCommits; @@ -137,13 +142,12 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag { if (_currentCommit is null) { - throw new InvalidAsynchronousStateException( - $"{nameof(_currentCommit)} is NULL when calling {nameof(Commit)}"); + ThrowInvalidAsynchronousStateException(); } if (!_allowCommits) { - throw new TrieException("Commits are not allowed on this trie."); + ThrowTrieException(); } if (RootRef is not null && RootRef.IsDirty) @@ -151,7 +155,7 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag Commit(new NodeCommitInfo(RootRef), skipSelf: skipRoot); while (_currentCommit.TryDequeue(out NodeCommitInfo node)) { - if (_logger.IsTrace) _logger.Trace($"Committing {node} in {blockNumber}"); + if (_isTrace) Trace(blockNumber, node); TrieStore.CommitNode(blockNumber, node, writeFlags: writeFlags); } @@ -161,21 +165,46 @@ public void Commit(long blockNumber, bool skipRoot = false, WriteFlags writeFlag } TrieStore.FinishBlockCommit(TrieType, blockNumber, RootRef, writeFlags); - if (_logger.IsDebug) _logger.Debug($"Finished committing block {blockNumber}"); + + if (_isDebug) Debug(blockNumber); + + [MethodImpl(MethodImplOptions.NoInlining)] + void Trace(long blockNumber, in NodeCommitInfo node) + { + _logger.Trace($"Committing {node} in {blockNumber}"); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + void Debug(long blockNumber) + { + _logger.Debug($"Finished committing block {blockNumber}"); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowInvalidAsynchronousStateException() + { + throw new InvalidAsynchronousStateException($"{nameof(_currentCommit)} is NULL when calling {nameof(Commit)}"); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowTrieException() + { + throw new TrieException("Commits are not allowed on this trie."); + } } private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) { if (_currentCommit is null) { - throw new InvalidAsynchronousStateException( - $"{nameof(_currentCommit)} is NULL when calling {nameof(Commit)}"); + ThrowInvalidAsynchronousStateException(nameof(_currentCommit)); } if (_commitExceptions is null) { - throw new InvalidAsynchronousStateException( - $"{nameof(_commitExceptions)} is NULL when calling {nameof(Commit)}"); + ThrowInvalidAsynchronousStateException(nameof(_commitExceptions)); } TrieNode node = nodeCommitInfo.Node; @@ -192,20 +221,16 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) } else { - if (_logger.IsTrace) + if (_isTrace) { - TrieNode child = node.GetChild(TrieStore, i); - if (child is not null) - { - _logger.Trace($"Skipping commit of {child}"); - } + Trace(node, i); } } } } else { - List nodesToCommit = new(); + List nodesToCommit = new(16); for (int i = 0; i < 16; i++) { if (node.IsChildDirty(i)) @@ -214,13 +239,9 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) } else { - if (_logger.IsTrace) + if (_isTrace) { - TrieNode child = node.GetChild(TrieStore, i); - if (child is not null) - { - _logger.Trace($"Skipping commit of {child}"); - } + Trace(node, i); } } } @@ -242,7 +263,7 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) if (!_commitExceptions.IsEmpty) { - throw new AggregateException(_commitExceptions); + ThrowAggregateExceptions(); } } else @@ -259,7 +280,7 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) TrieNode extensionChild = node.GetChild(TrieStore, 0); if (extensionChild is null) { - throw new InvalidOperationException("An attempt to store an extension without a child."); + ThrowInvalidExtension(); } if (extensionChild.IsDirty) @@ -268,7 +289,7 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) } else { - if (_logger.IsTrace) _logger.Trace($"Skipping commit of {extensionChild}"); + if (_isTrace) TraceExtensionSkip(extensionChild); } } @@ -284,7 +305,50 @@ private void Commit(NodeCommitInfo nodeCommitInfo, bool skipSelf = false) } else { - if (_logger.IsTrace) _logger.Trace($"Skipping commit of an inlined {node}"); + if (_isTrace) TraceSkipInlineNode(node); + } + + [DoesNotReturn] + [StackTraceHidden] + void ThrowAggregateExceptions() + { + throw new AggregateException(_commitExceptions); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowInvalidExtension() + { + throw new InvalidOperationException("An attempt to store an extension without a child."); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowInvalidAsynchronousStateException(string param) + { + throw new InvalidAsynchronousStateException($"{param} is NULL when calling {nameof(Commit)}"); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + void Trace(TrieNode node, int i) + { + TrieNode child = node.GetChild(TrieStore, i); + if (child is not null) + { + _logger.Trace($"Skipping commit of {child}"); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + void TraceExtensionSkip(TrieNode extensionChild) + { + _logger.Trace($"Skipping commit of {extensionChild}"); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + void TraceSkipInlineNode(TrieNode node) + { + _logger.Trace($"Skipping commit of an inlined {node}"); } } @@ -320,15 +384,11 @@ private void SetRootHash(Hash256? value, bool resetObjects) : array = ArrayPool.Shared.Rent(nibblesCount)) [..nibblesCount]; // Slice to exact size; - try - { - Nibbles.BytesToNibbleBytes(rawKey, nibbles); - return Run(nibbles, nibblesCount, new CappedArray(Array.Empty()), false, startRootHash: rootHash).ToArray(); - } - finally - { - if (array is not null) ArrayPool.Shared.Return(array); - } + Nibbles.BytesToNibbleBytes(rawKey, nibbles); + var result = Run(nibbles, nibblesCount, new CappedArray(Array.Empty()), false, startRootHash: rootHash).ToArray(); + if (array is not null) ArrayPool.Shared.Return(array); + + return result; } catch (TrieException e) { @@ -337,6 +397,7 @@ private void SetRootHash(Hash256? value, bool resetObjects) } } + [MethodImpl(MethodImplOptions.NoInlining)] private static void EnhanceException(ReadOnlySpan rawKey, ValueHash256 rootHash, TrieException baseException) { static TrieNodeException? GetTrieNodeException(TrieException? exception) => @@ -367,8 +428,7 @@ public virtual void Set(ReadOnlySpan rawKey, byte[] value) [DebuggerStepThrough] public virtual void Set(ReadOnlySpan rawKey, CappedArray value) { - if (_logger.IsTrace) - _logger.Trace($"{(value.Length == 0 ? $"Deleting {rawKey.ToHexString()}" : $"Setting {rawKey.ToHexString()} = {value.AsSpan().ToHexString()}")}"); + if (_isTrace) Trace(in rawKey, in value); int nibblesCount = 2 * rawKey.Length; byte[] array = null; @@ -377,14 +437,14 @@ public virtual void Set(ReadOnlySpan rawKey, CappedArray value) : array = ArrayPool.Shared.Rent(nibblesCount)) [..nibblesCount]; // Slice to exact size - try - { - Nibbles.BytesToNibbleBytes(rawKey, nibbles); - Run(nibbles, nibblesCount, value, true); - } - finally + Nibbles.BytesToNibbleBytes(rawKey, nibbles); + Run(nibbles, nibblesCount, value, true); + + if (array is not null) ArrayPool.Shared.Return(array); + + void Trace(in ReadOnlySpan rawKey, in CappedArray value) { - if (array is not null) ArrayPool.Shared.Return(array); + _logger.Trace($"{(value.Length == 0 ? $"Deleting {rawKey.ToHexString()}" : $"Setting {rawKey.ToHexString()} = {value.AsSpan().ToHexString()}")}"); } } @@ -404,7 +464,7 @@ private CappedArray Run( { if (isUpdate && startRootHash is not null) { - throw new InvalidOperationException("Only reads can be done in parallel on the Patricia tree"); + ThrowNonConcurrentWrites(); } #if DEBUG @@ -426,7 +486,7 @@ private CappedArray Run( CappedArray result; if (startRootHash is not null) { - if (_logger.IsTrace) _logger.Trace($"Starting from {startRootHash} - {traverseContext.ToString()}"); + if (_isTrace) TraceStart(startRootHash, in traverseContext); TrieNode startNode = TrieStore.FindCachedOrUnknown(startRootHash); ResolveNode(startNode, in traverseContext); result = TraverseNode(startNode, in traverseContext); @@ -438,23 +498,50 @@ private CappedArray Run( { if (traverseContext.UpdateValue.IsNotNull) { - if (_logger.IsTrace) _logger.Trace($"Setting new leaf node with value {traverseContext.UpdateValue}"); + if (_isTrace) TraceNewLeaf(in traverseContext); byte[] key = updatePath[..nibblesCount].ToArray(); RootRef = TrieNodeFactory.CreateLeaf(key, traverseContext.UpdateValue); } - if (_logger.IsTrace) _logger.Trace($"Keeping the root as null in {traverseContext.ToString()}"); + if (_isTrace) TraceNull(in traverseContext); result = traverseContext.UpdateValue; } else { ResolveNode(RootRef, in traverseContext); - if (_logger.IsTrace) _logger.Trace($"{traverseContext.ToString()}"); + if (_isTrace) TraceNode(in traverseContext); result = TraverseNode(RootRef, in traverseContext); } } return result; + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowNonConcurrentWrites() + { + throw new InvalidOperationException("Only reads can be done in parallel on the Patricia tree"); + } + + void TraceStart(Hash256 startRootHash, in TraverseContext traverseContext) + { + _logger.Trace($"Starting from {startRootHash} - {traverseContext.ToString()}"); + } + + void TraceNewLeaf(in TraverseContext traverseContext) + { + _logger.Trace($"Setting new leaf node with value {traverseContext.UpdateValue}"); + } + + void TraceNull(in TraverseContext traverseContext) + { + _logger.Trace($"Keeping the root as null in {traverseContext.ToString()}"); + } + + void TraceNode(in TraverseContext traverseContext) + { + _logger.Trace($"{traverseContext.ToString()}"); + } } private void ResolveNode(TrieNode node, in TraverseContext traverseContext) @@ -471,20 +558,36 @@ private void ResolveNode(TrieNode node, in TraverseContext traverseContext) private CappedArray TraverseNode(TrieNode node, in TraverseContext traverseContext) { - if (_logger.IsTrace) - _logger.Trace( - $"Traversing {node} to {(traverseContext.IsRead ? "READ" : traverseContext.IsDelete ? "DELETE" : "UPDATE")}"); + if (_isTrace) Trace(node, traverseContext); return node.NodeType switch { NodeType.Branch => TraverseBranch(node, in traverseContext), NodeType.Extension => TraverseExtension(node, in traverseContext), NodeType.Leaf => TraverseLeaf(node, in traverseContext), - NodeType.Unknown => throw new InvalidOperationException( - $"Cannot traverse unresolved node {node.Keccak}"), - _ => throw new NotSupportedException( - $"Unknown node type {node.NodeType}") + NodeType.Unknown => TraverseUnknown(node), + _ => ThrowNotSupported(node) }; + + [MethodImpl(MethodImplOptions.NoInlining)] + void Trace(TrieNode node, in TraverseContext traverseContext) + { + _logger.Trace($"Traversing {node} to {(traverseContext.IsRead ? "READ" : traverseContext.IsDelete ? "DELETE" : "UPDATE")}"); + } + + [DoesNotReturn] + [StackTraceHidden] + static CappedArray TraverseUnknown(TrieNode node) + { + throw new InvalidOperationException($"Cannot traverse unresolved node {node.Keccak}"); + } + + [DoesNotReturn] + [StackTraceHidden] + static CappedArray ThrowNotSupported(TrieNode node) + { + throw new NotSupportedException($"Unknown node type {node.NodeType}"); + } } private void ConnectNodes(TrieNode? node, in TraverseContext traverseContext) @@ -525,7 +628,7 @@ private void ConnectNodes(TrieNode? node, in TraverseContext traverseContext) // which is not possible in the Ethereum protocol where keys are of equal lengths // (it is possible in the more general trie definition) TrieNode leafFromBranch = TrieNodeFactory.CreateLeaf(Array.Empty(), node.Value); - if (_logger.IsTrace) _logger.Trace($"Converting {node} into {leafFromBranch}"); + if (_isTrace) _logger.Trace($"Converting {node} into {leafFromBranch}"); nextNode = leafFromBranch; } else @@ -567,7 +670,7 @@ L X - - - - - - - - - - - - - - */ { TrieNode extensionFromBranch = TrieNodeFactory.CreateExtension(new[] { (byte)childNodeIndex }, childNode); - if (_logger.IsTrace) + if (_isTrace) _logger.Trace( $"Extending child {childNodeIndex} {childNode} of {node} into {extensionFromBranch}"); @@ -603,7 +706,7 @@ L L - - - - - - - - - - - - - - */ byte[] newKey = Bytes.Concat((byte)childNodeIndex, childNode.Key); TrieNode extendedExtension = childNode.CloneWithChangedKey(newKey); - if (_logger.IsTrace) + if (_isTrace) _logger.Trace( $"Extending child {childNodeIndex} {childNode} of {node} into {extendedExtension}"); nextNode = extendedExtension; @@ -613,14 +716,14 @@ L L - - - - - - - - - - - - - - */ byte[] newKey = Bytes.Concat((byte)childNodeIndex, childNode.Key); TrieNode extendedLeaf = childNode.CloneWithChangedKey(newKey); - if (_logger.IsTrace) + if (_isTrace) _logger.Trace( $"Extending branch child {childNodeIndex} {childNode} into {extendedLeaf}"); - if (_logger.IsTrace) _logger.Trace($"Decrementing ref on a leaf extended up to eat a branch {childNode}"); + if (_isTrace) _logger.Trace($"Decrementing ref on a leaf extended up to eat a branch {childNode}"); if (node.IsSealed) { - if (_logger.IsTrace) _logger.Trace($"Decrementing ref on a branch replaced by a leaf {node}"); + if (_isTrace) _logger.Trace($"Decrementing ref on a branch replaced by a leaf {node}"); } nextNode = extendedLeaf; @@ -644,7 +747,7 @@ L L - - - - - - - - - - - - - - */ { byte[] newKey = Bytes.Concat(node.Key, nextNode.Key); TrieNode extendedLeaf = nextNode.CloneWithChangedKey(newKey); - if (_logger.IsTrace) + if (_isTrace) _logger.Trace($"Combining {node} and {nextNode} into {extendedLeaf}"); nextNode = extendedLeaf; @@ -678,7 +781,7 @@ L L - - - - - - - - - - - - - - */ byte[] newKey = Bytes.Concat(node.Key, nextNode.Key); TrieNode extendedExtension = nextNode.CloneWithChangedKey(newKey); - if (_logger.IsTrace) + if (_isTrace) _logger.Trace($"Combining {node} and {nextNode} into {extendedExtension}"); nextNode = extendedExtension; @@ -690,7 +793,7 @@ L L - - - - - - - - - - - - - - */ node = node.Clone(); } - if (_logger.IsTrace) _logger.Trace($"Connecting {node} with {nextNode}"); + if (_isTrace) _logger.Trace($"Connecting {node} with {nextNode}"); node.SetChild(0, nextNode); nextNode = node; } @@ -815,7 +918,7 @@ private CappedArray TraverseLeaf(TrieNode node, in TraverseContext travers longerPathValue = node.Value; } - int extensionLength = FindCommonPrefixLength(shorterPath, longerPath); + int extensionLength = shorterPath.CommonPrefixLength(longerPath); if (extensionLength == shorterPath.Length && extensionLength == longerPath.Length) { if (traverseContext.IsRead) @@ -893,7 +996,7 @@ private CappedArray TraverseExtension(TrieNode node, in TraverseContext tr TrieNode originalNode = node; ReadOnlySpan remaining = traverseContext.GetRemainingUpdatePath(); - int extensionLength = FindCommonPrefixLength(remaining, node.Key); + int extensionLength = remaining.CommonPrefixLength(node.Key); if (extensionLength == node.Key.Length) { if (traverseContext.IsUpdate) @@ -978,18 +1081,6 @@ private CappedArray TraverseNext(in TraverseContext traverseContext, int e return TraverseNode(next, in newContext); } - private static int FindCommonPrefixLength(ReadOnlySpan shorterPath, ReadOnlySpan longerPath) - { - int commonPrefixLength = 0; - int maxLength = Math.Min(shorterPath.Length, longerPath.Length); - for (int i = 0; i < maxLength && shorterPath[i] == longerPath[i]; i++, commonPrefixLength++) - { - // just finding the common part of the path - } - - return commonPrefixLength; - } - private readonly ref struct TraverseContext { public CappedArray UpdateValue { get; } @@ -1077,21 +1168,23 @@ public void Accept(ITreeVisitor visitor, Hash256 rootHash, VisitingOptions? visi if (!rootHash.Equals(Keccak.EmptyTreeHash)) { rootRef = RootHash == rootHash ? RootRef : TrieStore.FindCachedOrUnknown(rootHash); - try - { - rootRef!.ResolveNode(TrieStore); - } - catch (TrieException) + if (!rootRef!.TryResolveNode(TrieStore)) { visitor.VisitMissingNode(rootHash, trieVisitContext); return; } } - ITrieNodeResolver resolver = TrieStore; + ReadFlags flags = visitor.ExtraReadFlag; if (visitor.IsFullDbScan) { - resolver = new TrieNodeResolverWithReadFlags(TrieStore, ReadFlags.HintCacheMiss); + flags |= ReadFlags.HintCacheMiss; + } + + ITrieNodeResolver resolver = TrieStore; + if (flags != ReadFlags.None) + { + resolver = new TrieNodeResolverWithReadFlags(TrieStore, flags); } visitor.VisitTree(rootHash, trieVisitContext); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolver.cs b/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolver.cs index ba5324457b5..3773cb26104 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolver.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ITrieNodeResolver.cs @@ -23,5 +23,12 @@ public interface ITrieNodeResolver /// /// byte[]? LoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None); + + /// + /// Loads RLP of the node. + /// + /// + /// + byte[]? TryLoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None); } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieNodeResolver.cs b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieNodeResolver.cs index 5eaa828eda1..5856ac64bb4 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieNodeResolver.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieNodeResolver.cs @@ -14,5 +14,6 @@ private NullTrieNodeResolver() { } public TrieNode FindCachedOrUnknown(Hash256 hash) => new(NodeType.Unknown, hash); public byte[]? LoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None) => null; + public byte[]? TryLoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None) => null; } } diff --git a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs index 9b59b246232..55f77dc2b40 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/NullTrieStore.cs @@ -29,6 +29,8 @@ public event EventHandler ReorgBoundaryReached public TrieNode FindCachedOrUnknown(Hash256 hash) => new(NodeType.Unknown, hash); + public byte[] TryLoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None) => null; + public byte[] LoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None) => Array.Empty(); public bool IsPersisted(in ValueHash256 keccak) => true; diff --git a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs index f5ccf20926d..86710b26720 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/ReadOnlyTrieStore.cs @@ -26,6 +26,7 @@ public ReadOnlyTrieStore(TrieStore trieStore, IKeyValueStore? readOnlyStore) public TrieNode FindCachedOrUnknown(Hash256 hash) => _trieStore.FindCachedOrUnknown(hash, true); + public byte[]? TryLoadRlp(Hash256 hash, ReadFlags flags) => _trieStore.TryLoadRlp(hash, _readOnlyStore, flags); public byte[] LoadRlp(Hash256 hash, ReadFlags flags) => _trieStore.LoadRlp(hash, _readOnlyStore, flags); public bool IsPersisted(in ValueHash256 keccak) => _trieStore.IsPersisted(keccak); diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 38c607dbf30..ec18d0c4ec6 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -322,22 +322,32 @@ public void FinishBlockCommit(TrieType trieType, long blockNumber, TrieNode? roo public event EventHandler? ReorgBoundaryReached; - public byte[] LoadRlp(Hash256 keccak, IKeyValueStore? keyValueStore, ReadFlags readFlags = ReadFlags.None) + public byte[]? TryLoadRlp(Hash256 keccak, IKeyValueStore? keyValueStore, ReadFlags readFlags = ReadFlags.None) { keyValueStore ??= _keyValueStore; byte[]? rlp = keyValueStore.Get(keccak.Bytes, readFlags); + if (rlp is not null) + { + Metrics.LoadedFromDbNodesCount++; + } + + return rlp; + } + + public byte[] LoadRlp(Hash256 keccak, IKeyValueStore? keyValueStore, ReadFlags readFlags = ReadFlags.None) + { + byte[]? rlp = TryLoadRlp(keccak, keyValueStore, readFlags); if (rlp is null) { throw new TrieNodeException($"Node {keccak} is missing from the DB", keccak); } - Metrics.LoadedFromDbNodesCount++; - return rlp; } public virtual byte[] LoadRlp(Hash256 keccak, ReadFlags readFlags = ReadFlags.None) => LoadRlp(keccak, null, readFlags); + public virtual byte[]? TryLoadRlp(Hash256 keccak, ReadFlags readFlags = ReadFlags.None) => TryLoadRlp(keccak, null, readFlags); public bool IsPersisted(in ValueHash256 keccak) { diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs index a8bf54f78ba..5a87b3c1e1c 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.Decoder.cs @@ -4,7 +4,7 @@ using System; using System.Buffers; using System.Diagnostics; -using System.Linq; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using Nethermind.Core.Buffers; using Nethermind.Core.Crypto; @@ -29,7 +29,7 @@ public static CappedArray Encode(ITrieNodeResolver tree, TrieNode? item, I if (item is null) { - throw new TrieException("An attempt was made to RLP encode a null node."); + ThrowNullNode(); } return item.NodeType switch @@ -37,8 +37,22 @@ public static CappedArray Encode(ITrieNodeResolver tree, TrieNode? item, I NodeType.Branch => RlpEncodeBranch(tree, item, bufferPool), NodeType.Extension => EncodeExtension(tree, item, bufferPool), NodeType.Leaf => EncodeLeaf(item, bufferPool), - _ => throw new TrieException($"An attempt was made to encode a trie node of type {item.NodeType}") + _ => ThrowUnhandledNodeType(item) }; + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowNullNode() + { + throw new TrieException("An attempt was made to RLP encode a null node."); + } + + [DoesNotReturn] + [StackTraceHidden] + static CappedArray ThrowUnhandledNodeType(TrieNode item) + { + throw new TrieException($"An attempt was made to encode a trie node of type {item.NodeType}"); + } } [SkipLocalsInit] @@ -101,7 +115,7 @@ private static CappedArray EncodeLeaf(TrieNode node, ICappedArrayPool? poo { if (node.Key is null) { - throw new TrieException($"Hex prefix of a leaf node is null at node {node.Keccak}"); + ThrowNullKey(node); } byte[] hexPrefix = node.Key; @@ -130,6 +144,13 @@ private static CappedArray EncodeLeaf(TrieNode node, ICappedArrayPool? poo return data; } + [DoesNotReturn] + [StackTraceHidden] + private static void ThrowNullKey(TrieNode node) + { + throw new TrieException($"Hex prefix of a leaf node is null at node {node.Keccak}"); + } + private static CappedArray RlpEncodeBranch(ITrieNodeResolver tree, TrieNode item, ICappedArrayPool? pool) { int valueRlpLength = AllowBranchValues ? Rlp.LengthOf(item.Value.AsSpan()) : 1; diff --git a/src/Nethermind/Nethermind.Trie/TrieNode.cs b/src/Nethermind/Nethermind.Trie/TrieNode.cs index f65a5decc5a..281c326fca1 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNode.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNode.cs @@ -3,6 +3,8 @@ using System; using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Threading; using Nethermind.Core; @@ -73,13 +75,19 @@ internal set { if (IsSealed) { - throw new InvalidOperationException( - $"{nameof(TrieNode)} {this} is already sealed when setting {nameof(Key)}."); + ThrowAlreadySealed(); } InitData(); _data![0] = value; Keccak = null; + + [DoesNotReturn] + [StackTraceHidden] + void ThrowAlreadySealed() + { + throw new InvalidOperationException($"{nameof(TrieNode)} {this} is already sealed when setting {nameof(Key)}."); + } } } @@ -144,8 +152,7 @@ public CappedArray Value { if (IsSealed) { - throw new InvalidOperationException( - $"{nameof(TrieNode)} {this} is already sealed when setting {nameof(Value)}."); + ThrowAlreadySealed(); } InitData(); @@ -170,6 +177,13 @@ public CappedArray Value } _data![IsLeaf ? 1 : BranchesCount] = value; + + [DoesNotReturn] + [StackTraceHidden] + void ThrowAlreadySealed() + { + throw new InvalidOperationException($"{nameof(TrieNode)} {this} is already sealed when setting {nameof(Value)}."); + } } } @@ -262,10 +276,17 @@ public void Seal() { if (IsSealed) { - throw new InvalidOperationException($"{nameof(TrieNode)} {this} is already sealed."); + ThrowAlreadySealed(); } IsDirty = false; + + [DoesNotReturn] + [StackTraceHidden] + void ThrowAlreadySealed() + { + throw new InvalidOperationException($"{nameof(TrieNode)} {this} is already sealed."); + } } /// @@ -281,7 +302,7 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. { if (Keccak is null) { - throw new TrieException("Unable to resolve node without Keccak"); + ThrowMissingKeccak(); } FullRlp = tree.LoadRlp(Keccak, readFlags); @@ -289,7 +310,7 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. if (FullRlp.IsNull) { - throw new TrieException($"Trie returned a NULL RLP for node {Keccak}"); + ThrowNullRlp(); } } } @@ -301,55 +322,152 @@ public void ResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags. _rlpStream = FullRlp.AsRlpStream(); if (_rlpStream is null) { - throw new InvalidAsynchronousStateException($"{nameof(_rlpStream)} is null when {nameof(NodeType)} is {NodeType}"); + ThrowInvalidStateException(); + return; + } + + if (!DecodeRlp(bufferPool, out int numberOfItems)) + { + ThrowUnexpectedNumberOfItems(numberOfItems); } + } + catch (RlpException rlpException) + { + ThrowDecodingError(rlpException); + } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowMissingKeccak() + { + throw new TrieException("Unable to resolve node without Keccak"); + } - Metrics.TreeNodeRlpDecodings++; - _rlpStream.ReadSequenceLength(); + [DoesNotReturn] + [StackTraceHidden] + void ThrowNullRlp() + { + throw new TrieException($"Trie returned a NULL RLP for node {Keccak}"); + } - // micro optimization to prevent searches beyond 3 items for branches (search up to three) - int numberOfItems = _rlpStream.PeekNumberOfItemsRemaining(null, 3); + [DoesNotReturn] + [StackTraceHidden] + void ThrowInvalidStateException() + { + throw new InvalidAsynchronousStateException($"{nameof(_rlpStream)} is null when {nameof(NodeType)} is {NodeType}"); + } - if (numberOfItems > 2) + [DoesNotReturn] + [StackTraceHidden] + void ThrowUnexpectedNumberOfItems(int numberOfItems) + { + throw new TrieNodeException($"Unexpected number of items = {numberOfItems} when decoding a node from RLP ({FullRlp.AsSpan().ToHexString()})", Keccak ?? Nethermind.Core.Crypto.Keccak.Zero); + } + + [DoesNotReturn] + [StackTraceHidden] + void ThrowDecodingError(RlpException rlpException) + { + throw new TrieNodeException($"Error when decoding node {Keccak}", Keccak ?? Nethermind.Core.Crypto.Keccak.Zero, rlpException); + } + } + + /// + /// Highly optimized + /// + public bool TryResolveNode(ITrieNodeResolver tree, ReadFlags readFlags = ReadFlags.None, ICappedArrayPool? bufferPool = null) + { + try + { + if (NodeType == NodeType.Unknown) + { + if (FullRlp.IsNull) + { + if (Keccak is null) + { + return false; + } + + FullRlp = tree.TryLoadRlp(Keccak, readFlags); + IsPersisted = true; + + if (FullRlp.IsNull) + { + return false; + } + } + } + else { - NodeType = NodeType.Branch; + return true; } - else if (numberOfItems == 2) + + _rlpStream = FullRlp.AsRlpStream(); + if (_rlpStream is null) { - (byte[] key, bool isLeaf) = HexPrefix.FromBytes(_rlpStream.DecodeByteArraySpan()); + ThrowInvalidStateException(); + } - // a hack to set internally and still verify attempts from the outside - // after the code is ready we should just add proper access control for methods from the outside and inside - bool isDirtyActual = IsDirty; - IsDirty = true; + return DecodeRlp(bufferPool, out _); + } + catch (RlpException) + { + return false; + } - if (isLeaf) - { - NodeType = NodeType.Leaf; - Key = key; + [DoesNotReturn] + [StackTraceHidden] + void ThrowInvalidStateException() + { + throw new InvalidAsynchronousStateException($"{nameof(_rlpStream)} is null when {nameof(NodeType)} is {NodeType}"); + } + } - ReadOnlySpan valueSpan = _rlpStream.DecodeByteArraySpan(); - CappedArray buffer = bufferPool.SafeRentBuffer(valueSpan.Length); - valueSpan.CopyTo(buffer.AsSpan()); - Value = buffer; - } - else - { - NodeType = NodeType.Extension; - Key = key; - } + private bool DecodeRlp(ICappedArrayPool bufferPool, out int itemsCount) + { + Metrics.TreeNodeRlpDecodings++; + _rlpStream.ReadSequenceLength(); + + // micro optimization to prevent searches beyond 3 items for branches (search up to three) + int numberOfItems = itemsCount = _rlpStream.PeekNumberOfItemsRemaining(null, 3); + + if (numberOfItems > 2) + { + NodeType = NodeType.Branch; + } + else if (numberOfItems == 2) + { + (byte[] key, bool isLeaf) = HexPrefix.FromBytes(_rlpStream.DecodeByteArraySpan()); - IsDirty = isDirtyActual; + // a hack to set internally and still verify attempts from the outside + // after the code is ready we should just add proper access control for methods from the outside and inside + bool isDirtyActual = IsDirty; + IsDirty = true; + + if (isLeaf) + { + NodeType = NodeType.Leaf; + Key = key; + + ReadOnlySpan valueSpan = _rlpStream.DecodeByteArraySpan(); + CappedArray buffer = bufferPool.SafeRentBuffer(valueSpan.Length); + valueSpan.CopyTo(buffer.AsSpan()); + Value = buffer; } else { - throw new TrieNodeException($"Unexpected number of items = {numberOfItems} when decoding a node from RLP ({FullRlp.AsSpan().ToHexString()})", Keccak ?? Nethermind.Core.Crypto.Keccak.Zero); + NodeType = NodeType.Extension; + Key = key; } + + IsDirty = isDirtyActual; } - catch (RlpException rlpException) + else { - throw new TrieNodeException($"Error when decoding node {Keccak}", Keccak ?? Nethermind.Core.Crypto.Keccak.Zero, rlpException); + return false; } + + return true; } public void ResolveKey(ITrieNodeResolver tree, bool isRoot, ICappedArrayPool? bufferPool = null) @@ -475,8 +593,7 @@ public bool IsChildNull(int i) { if (!IsBranch) { - throw new TrieException( - "An attempt was made to ask about whether a child is null on a non-branch node."); + ThrowNotABranch(); } if (_rlpStream is not null && _data?[i] is null) @@ -486,6 +603,13 @@ public bool IsChildNull(int i) } return _data?[i] is null || ReferenceEquals(_data[i], _nullNode); + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowNotABranch() + { + throw new TrieException("An attempt was made to ask about whether a child is null on a non-branch node."); + } } public bool IsChildDirty(int i) @@ -544,9 +668,7 @@ public TrieNode? this[int i] { // we expect this to happen as a Trie traversal error (please see the stack trace above) // we need to investigate this case when it happens again - bool isKeccakCalculated = Keccak is not null && FullRlp.IsNotNull; - bool isKeccakCorrect = isKeccakCalculated && Keccak == Nethermind.Core.Crypto.Keccak.Compute(FullRlp.AsSpan()); - throw new TrieException($"Unexpected type found at position {childIndex} of {this} with {nameof(_data)} of length {_data?.Length}. Expected a {nameof(TrieNode)} or {nameof(Keccak)} but found {childOrRef?.GetType()} with a value of {childOrRef}. Keccak calculated? : {isKeccakCalculated}; Keccak correct? : {isKeccakCorrect}"); + ThrowUnexpectedTypeException(childIndex, childOrRef); } // pruning trick so we never store long persisted paths @@ -556,6 +678,15 @@ public TrieNode? this[int i] } return child; + + [DoesNotReturn] + [StackTraceHidden] + void ThrowUnexpectedTypeException(int childIndex, object childOrRef) + { + bool isKeccakCalculated = Keccak is not null && FullRlp.IsNotNull; + bool isKeccakCorrect = isKeccakCalculated && Keccak == Nethermind.Core.Crypto.Keccak.Compute(FullRlp.AsSpan()); + throw new TrieException($"Unexpected type found at position {childIndex} of {this} with {nameof(_data)} of length {_data?.Length}. Expected a {nameof(TrieNode)} or {nameof(Keccak)} but found {childOrRef?.GetType()} with a value of {childOrRef}. Keccak calculated? : {isKeccakCalculated}; Keccak correct? : {isKeccakCorrect}"); + } } public void ReplaceChildRef(int i, TrieNode child) @@ -574,14 +705,20 @@ public void SetChild(int i, TrieNode? node) { if (IsSealed) { - throw new InvalidOperationException( - $"{nameof(TrieNode)} {this} is already sealed when setting a child."); + ThrowAlreadySealed(); } InitData(); int index = IsExtension ? i + 1 : i; _data![index] = node ?? _nullNode; Keccak = null; + + [DoesNotReturn] + [StackTraceHidden] + void ThrowAlreadySealed() + { + throw new InvalidOperationException($"{nameof(TrieNode)} {this} is already sealed when setting a child."); + } } public long GetMemorySize(bool recursive) @@ -803,8 +940,6 @@ public void PrunePersistedRecursively(int maxLevelsDeep) // } } - #region private - private bool TryResolveStorageRoot(ITrieNodeResolver resolver, out TrieNode? storageRoot) { bool hasStorage = false; @@ -837,8 +972,8 @@ private void InitData() switch (NodeType) { case NodeType.Unknown: - throw new InvalidOperationException( - $"Cannot resolve children of an {nameof(NodeType.Unknown)} node"); + ThrowCannotResolveException(); + return; case NodeType.Branch: _data = new object[AllowBranchValues ? BranchesCount + 1 : BranchesCount]; break; @@ -847,6 +982,13 @@ private void InitData() break; } } + + [DoesNotReturn] + [StackTraceHidden] + static void ThrowCannotResolveException() + { + throw new InvalidOperationException($"Cannot resolve children of an {nameof(NodeType.Unknown)} node"); + } } private void SeekChild(int itemToSetOn) @@ -945,7 +1087,7 @@ private void UnresolveChild(int i) { if (!childNode.IsPersisted) { - throw new InvalidOperationException("Cannot unresolve a child that is not persisted yet."); + ThrowNotPersisted(); } else if (childNode.Keccak is not null) // if not by value node { @@ -953,8 +1095,13 @@ private void UnresolveChild(int i) } } } - } - #endregion + [DoesNotReturn] + [StackTraceHidden] + static void ThrowNotPersisted() + { + throw new InvalidOperationException("Cannot unresolve a child that is not persisted yet."); + } + } } } diff --git a/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs b/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs index b417aaf5e70..5f96323bf22 100644 --- a/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs +++ b/src/Nethermind/Nethermind.Trie/TrieNodeResolverWithReadFlags.cs @@ -23,6 +23,16 @@ public TrieNode FindCachedOrUnknown(Hash256 hash) return _baseResolver.FindCachedOrUnknown(hash); } + public byte[]? TryLoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None) + { + if (flags != ReadFlags.None) + { + return _baseResolver.TryLoadRlp(hash, flags | _defaultFlags); + } + + return _baseResolver.TryLoadRlp(hash, _defaultFlags); + } + public byte[]? LoadRlp(Hash256 hash, ReadFlags flags = ReadFlags.None) { if (flags != ReadFlags.None) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index f19a671aa60..f4d2b2092ae 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -6,6 +6,7 @@ using Nethermind.Core; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; using Nethermind.Evm; using Nethermind.Int256; using NUnit.Framework; @@ -18,7 +19,7 @@ public partial class TxPoolTests [Test] public void should_reject_blob_tx_if_blobs_not_supported([Values(true, false)] bool isBlobSupportEnabled) { - TxPoolConfig txPoolConfig = new() { BlobSupportEnabled = isBlobSupportEnabled }; + TxPoolConfig txPoolConfig = new() { BlobsSupport = isBlobSupportEnabled ? BlobsSupportMode.InMemory : BlobsSupportMode.Disabled }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); Transaction tx = Build.A.Transaction @@ -40,8 +41,7 @@ public void blob_pool_size_should_be_correct([Values(true, false)] bool persiste const int poolSize = 10; TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, - PersistentBlobStorageEnabled = persistentStorageEnabled, + BlobsSupport = persistentStorageEnabled ? BlobsSupportMode.Storage : BlobsSupportMode.InMemory, PersistentBlobStorageSize = persistentStorageEnabled ? poolSize : 0, InMemoryBlobPoolSize = persistentStorageEnabled ? 0 : poolSize }; @@ -73,7 +73,7 @@ public void should_reject_txs_with_nonce_too_far_in_future(TxType txType, int ma { TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, + BlobsSupport = BlobsSupportMode.InMemory, Size = 100, MaxPendingTxsPerSender = maxPendingTxs, MaxPendingBlobTxsPerSender = maxPendingBlobTxs @@ -102,9 +102,8 @@ public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, f const int poolSize = 10; TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, + BlobsSupport = persistentStorageEnabled ? BlobsSupportMode.Storage : BlobsSupportMode.InMemory, Size = isBlob ? 0 : poolSize, - PersistentBlobStorageEnabled = persistentStorageEnabled, PersistentBlobStorageSize = persistentStorageEnabled ? poolSize : 0, InMemoryBlobPoolSize = persistentStorageEnabled ? 0 : poolSize }; @@ -144,9 +143,8 @@ public void should_add_blob_tx_and_return_when_requested([Values(true, false)] b { TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, - Size = 10, - PersistentBlobStorageEnabled = isPersistentStorage + BlobsSupport = isPersistentStorage ? BlobsSupportMode.Storage : BlobsSupportMode.InMemory, + Size = 10 }; BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); @@ -191,7 +189,7 @@ public void should_not_throw_when_asking_for_non_existing_tx() [TestCase(1_000_000_000, true)] public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than_1GWei(int maxPriorityFeePerGas, bool expectedResult) { - TxPoolConfig txPoolConfig = new() { BlobSupportEnabled = true, Size = 10 }; + TxPoolConfig txPoolConfig = new() { BlobsSupport = BlobsSupportMode.InMemory, Size = 10 }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -209,7 +207,7 @@ public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than [Test] public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) { - _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction @@ -246,7 +244,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } - _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); @@ -261,9 +259,8 @@ public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() { TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, - Size = 10, - PersistentBlobStorageEnabled = true + BlobsSupport = BlobsSupportMode.Storage, + Size = 10 }; BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); @@ -310,9 +307,8 @@ public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_st { TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, - Size = 10, - PersistentBlobStorageEnabled = isPersistentStorage + BlobsSupport = isPersistentStorage ? BlobsSupportMode.Storage : BlobsSupportMode.InMemory, + Size = 10 }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -338,9 +334,8 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerBlobGas_is_ { TxPoolConfig txPoolConfig = new() { - BlobSupportEnabled = true, - Size = 10, - PersistentBlobStorageEnabled = isPersistentStorage + BlobsSupport = isPersistentStorage ? BlobsSupportMode.Storage : BlobsSupportMode.InMemory, + Size = 10 }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -380,7 +375,7 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, { bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; - _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction @@ -413,7 +408,7 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, [Test] public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, true)] bool supportsBlobs) { - _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -455,7 +450,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } - _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); @@ -494,14 +489,124 @@ public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int ex [Test] public void RecoverAddress_should_work_correctly() { - Transaction tx = Build.A.Transaction + Transaction tx = GetTx(TestItem.PrivateKeyA); + _ethereumEcdsa.RecoverAddress(tx).Should().Be(tx.SenderAddress); + } + + [Test] + public async Task should_add_processed_txs_to_db() + { + const long blockNumber = 358; + + BlobTxStorage blobTxStorage = new(); + ITxPoolConfig txPoolConfig = new TxPoolConfig() + { + Size = 128, + BlobsSupport = BlobsSupportMode.StorageWithReorgs + }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); + + Transaction[] txs = { GetTx(TestItem.PrivateKeyA), GetTx(TestItem.PrivateKeyB) }; + + _txPool.SubmitTx(txs[0], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(txs[1], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(txs.Length); + _stateProvider.IncrementNonce(TestItem.AddressA); + _stateProvider.IncrementNonce(TestItem.AddressB); + + Block block = Build.A.Block.WithNumber(blockNumber).WithTransactions(txs).TestObject; + + await RaiseBlockAddedToMainAndWaitForTransactions(txs.Length, block); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(0); + + blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out Transaction[] returnedTxs).Should().BeTrue(); + returnedTxs.Length.Should().Be(txs.Length); + returnedTxs.Should().BeEquivalentTo(txs, options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex) // ...as well as PoolIndex + .Excluding(t => t.SenderAddress)); // sender is recovered later, it is not returned from db + + blobTxStorage.DeleteBlobTransactionsFromBlock(blockNumber); + blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out returnedTxs).Should().BeFalse(); + } + + [Test] + public async Task should_bring_back_reorganized_blob_txs() + { + const long blockNumber = 358; + + BlobTxStorage blobTxStorage = new(); + ITxPoolConfig txPoolConfig = new TxPoolConfig() + { + Size = 128, + BlobsSupport = BlobsSupportMode.StorageWithReorgs + }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressC, UInt256.MaxValue); + + Transaction[] txsA = { GetTx(TestItem.PrivateKeyA), GetTx(TestItem.PrivateKeyB) }; + Transaction[] txsB = { GetTx(TestItem.PrivateKeyC) }; + + _txPool.SubmitTx(txsA[0], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(txsA[1], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(txsB[0], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(txsA.Length + txsB.Length); + + // adding block A + Block blockA = Build.A.Block.WithNumber(blockNumber).WithTransactions(txsA).TestObject; + await RaiseBlockAddedToMainAndWaitForTransactions(txsA.Length, blockA); + + _txPool.GetPendingBlobTransactionsCount().Should().Be(txsB.Length); + _txPool.TryGetPendingBlobTransaction(txsA[0].Hash!, out _).Should().BeFalse(); + _txPool.TryGetPendingBlobTransaction(txsA[1].Hash!, out _).Should().BeFalse(); + _txPool.TryGetPendingBlobTransaction(txsB[0].Hash!, out _).Should().BeTrue(); + + // reorganized from block A to block B + Block blockB = Build.A.Block.WithNumber(blockNumber).WithTransactions(txsB).TestObject; + await RaiseBlockAddedToMainAndWaitForTransactions(txsB.Length + txsA.Length, blockB, blockA); + + // tx from block B should be removed from blob pool, but present in processed txs db + _txPool.TryGetPendingBlobTransaction(txsB[0].Hash!, out _).Should().BeFalse(); + blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out Transaction[] blockBTxs).Should().BeTrue(); + txsB.Should().BeEquivalentTo(blockBTxs, options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex) // ...as well as PoolIndex + .Excluding(t => t.SenderAddress)); // sender is recovered later, it is not returned from db + + // blob txs from reorganized blockA should be readded to blob pool + _txPool.GetPendingBlobTransactionsCount().Should().Be(txsA.Length); + _txPool.TryGetPendingBlobTransaction(txsA[0].Hash!, out Transaction tx1).Should().BeTrue(); + _txPool.TryGetPendingBlobTransaction(txsA[1].Hash!, out Transaction tx2).Should().BeTrue(); + + tx1.Should().BeEquivalentTo(txsA[0], options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex + + tx2.Should().BeEquivalentTo(txsA[1], options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex + } + + private Transaction GetTx(PrivateKey sender) + { + return Build.A.Transaction .WithShardBlobTxTypeAndFields() .WithMaxFeePerGas(1.GWei()) .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _ethereumEcdsa.RecoverAddress(tx).Should().Be(tx.SenderAddress); + .SignedAndResolved(_ethereumEcdsa, sender).TestObject; } } } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 6bd0ca3925f..e2555cd1bd1 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1657,7 +1657,7 @@ private TxPool CreatePool( ISpecProvider specProvider = null, ChainHeadInfoProvider chainHeadInfoProvider = null, IIncomingTxFilter incomingTxFilter = null, - ITxStorage txStorage = null, + IBlobTxStorage txStorage = null, bool thereIsPriorityContract = false) { specProvider ??= MainnetSpecProvider.Instance; @@ -1803,11 +1803,15 @@ private Transaction GetTransaction(UInt256 nonce, long gasLimit, UInt256 gasPric .SignedAndResolved(_ethereumEcdsa, privateKey) .TestObject; - private async Task RaiseBlockAddedToMainAndWaitForTransactions(int txCount) + private async Task RaiseBlockAddedToMainAndWaitForTransactions(int txCount, Block block = null, Block previousBlock = null) { + BlockReplacementEventArgs blockReplacementEventArgs = previousBlock is null + ? new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject) + : new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject, previousBlock); + SemaphoreSlim semaphoreSlim = new(0, txCount); _txPool.NewPending += (o, e) => semaphoreSlim.Release(); - _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(Build.A.Block.TestObject)); + _blockTree.BlockAddedToMain += Raise.EventWith(blockReplacementEventArgs); for (int i = 0; i < txCount; i++) { await semaphoreSlim.WaitAsync(10); diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 68f399b7fd2..4ba704d7c54 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -14,23 +14,25 @@ namespace Nethermind.TxPool; -public class BlobTxStorage : ITxStorage +public class BlobTxStorage : IBlobTxStorage { + private static readonly TxDecoder _txDecoder = new(); private readonly IDb _fullBlobTxsDb; private readonly IDb _lightBlobTxsDb; - private static readonly TxDecoder _txDecoder = new(); - private static readonly LightTxDecoder _lightTxDecoder = new(); + private readonly IDb _processedBlobTxsDb; public BlobTxStorage() { _fullBlobTxsDb = new MemDb(); _lightBlobTxsDb = new MemDb(); + _processedBlobTxsDb = new MemDb(); } public BlobTxStorage(IColumnsDb database) { _fullBlobTxsDb = database.GetColumnDb(BlobTxsColumns.FullBlobTxs); _lightBlobTxsDb = database.GetColumnDb(BlobTxsColumns.LightBlobTxs); + _processedBlobTxsDb = database.GetColumnDb(BlobTxsColumns.ProcessedTxs); } public bool TryGet(in ValueHash256 hash, Address sender, in UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) @@ -63,12 +65,7 @@ public void Add(Transaction transaction) Span txHashPrefixed = stackalloc byte[64]; GetHashPrefixedByTimestamp(transaction.Timestamp, transaction.Hash, txHashPrefixed); - int length = _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); - IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(length); - using NettyRlpStream rlpStream = new(byteBuffer); - rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); - - _fullBlobTxsDb.PutSpan(txHashPrefixed, byteBuffer.AsSpan()); + _fullBlobTxsDb.PutSpan(txHashPrefixed, EncodeTx(transaction)); _lightBlobTxsDb.Set(transaction.Hash, LightTxDecoder.Encode(transaction)); } @@ -81,6 +78,34 @@ public void Delete(in ValueHash256 hash, in UInt256 timestamp) _lightBlobTxsDb.Remove(hash.BytesAsSpan); } + public void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions) + { + if (blockBlobTransactions.Count == 0) + { + return; + } + + _processedBlobTxsDb.Set(blockNumber, EncodeTxs(blockBlobTransactions)); + } + + public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions) + { + byte[]? bytes = _processedBlobTxsDb.Get(blockNumber); + + if (bytes is not null) + { + RlpStream rlpStream = new(bytes); + blockBlobTransactions = _txDecoder.DecodeArray(rlpStream, RlpBehaviors.InMempoolForm); + return true; + } + + blockBlobTransactions = default; + return false; + } + + public void DeleteBlobTransactionsFromBlock(long blockNumber) + => _processedBlobTxsDb.Delete(blockNumber); + private static bool TryDecodeFullTx(byte[]? txBytes, Address sender, out Transaction? transaction) { if (txBytes is not null) @@ -112,6 +137,42 @@ private static void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueHash256 h timestamp.WriteBigEndian(txHashPrefixed); hash.Bytes.CopyTo(txHashPrefixed[32..]); } + + private Span EncodeTx(Transaction transaction) + { + int length = _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(length); + using NettyRlpStream rlpStream = new(byteBuffer); + rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); + + return byteBuffer.AsSpan(); + } + + private byte[] EncodeTxs(IList blockBlobTransactions) + { + int contentLength = GetLength(blockBlobTransactions); + + IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(Rlp.LengthOfSequence(contentLength)); + using NettyRlpStream rlpStream = new(byteBuffer); + rlpStream.StartSequence(contentLength); + foreach (Transaction transaction in blockBlobTransactions) + { + _txDecoder.Encode(rlpStream, transaction, RlpBehaviors.InMempoolForm); + } + + return byteBuffer.Array; + } + + private int GetLength(IList blockBlobTransactions) + { + int contentLength = 0; + foreach (Transaction transaction in blockBlobTransactions) + { + contentLength += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + } + + return contentLength; + } } internal static class UInt256Extensions diff --git a/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs b/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs new file mode 100644 index 00000000000..292706d5099 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.TxPool; + +/// +/// Defines blobs support mode. +/// +public enum BlobsSupportMode +{ + /// + /// No support for blob transactions. + /// + Disabled, + + /// + /// Blob transactions stored only in memory + /// + InMemory, + + /// + /// Blob transactions stored in db. + /// + Storage, + + /// + /// Blob transactions stored in db with support for restoring reorganized blob transactions to blob pool. + /// + StorageWithReorgs +} + +public static class BlobsSupportModeExtensions +{ + public static bool IsPersistentStorage(this BlobsSupportMode mode) => mode is BlobsSupportMode.Storage or BlobsSupportMode.StorageWithReorgs; + public static bool IsEnabled(this BlobsSupportMode mode) => mode is not BlobsSupportMode.Disabled; + public static bool IsDisabled(this BlobsSupportMode mode) => mode is BlobsSupportMode.Disabled; + public static bool SupportsReorgs(this BlobsSupportMode mode) => mode is BlobsSupportMode.StorageWithReorgs; +} diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index d77c44aeeca..96332932782 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -24,7 +24,7 @@ protected override IComparer GetReplacementComparer(IComparer _poolCapacity && _logger.IsWarn) + if (_logger.IsWarn && Count > _poolCapacity) _logger.Warn($"Blob pool exceeds the config size {Count}/{_poolCapacity}"); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs index e6071d4947f..948871ada72 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs @@ -49,7 +49,7 @@ protected override void InsertCore(TKey key, TValue value, TGroupKey groupKey) { if (_distinctDictionary.TryGetValue(value, out KeyValuePair oldKvp)) { - TryRemove(oldKvp.Key); + TryRemoveNonLocked(oldKvp.Key, evicted: false, out _, out _); } base.InsertCore(key, value, groupKey); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 578b29700b4..13243f06886 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Runtime.CompilerServices; using Nethermind.Core.Collections; +using Nethermind.Core.Threading; namespace Nethermind.TxPool.Collections { @@ -21,6 +22,8 @@ public abstract partial class SortedPool where TKey : notnull where TGroupKey : notnull { + protected McsPriorityLock Lock { get; } = new(); + private readonly int _capacity; // comparer for a bucket @@ -82,9 +85,10 @@ protected SortedPool(int capacity, IComparer comparer) /// /// Gets all items in random order. /// - [MethodImpl(MethodImplOptions.Synchronized)] public TValue[] GetSnapshot() { + using var lockRelease = Lock.Acquire(); + TValue[]? snapshot = _snapshot; snapshot ??= _snapshot = _buckets.SelectMany(b => b.Value).ToArray(); @@ -94,9 +98,10 @@ public TValue[] GetSnapshot() /// /// Gets all items in groups in supplied comparer order in groups. /// - [MethodImpl(MethodImplOptions.Synchronized)] public IDictionary GetBucketSnapshot(Predicate? where = null) { + using var lockRelease = Lock.Acquire(); + IEnumerable>> buckets = _buckets; if (where is not null) { @@ -108,9 +113,10 @@ public IDictionary GetBucketSnapshot(Predicate? /// /// Gets all items of requested group. /// - [MethodImpl(MethodImplOptions.Synchronized)] public TValue[] GetBucketSnapshot(TGroupKey group) { + using var lockRelease = Lock.Acquire(); + if (group is null) throw new ArgumentNullException(nameof(group)); return _buckets.TryGetValue(group, out EnhancedSortedSet? bucket) ? bucket.ToArray() : Array.Empty(); } @@ -118,9 +124,10 @@ public TValue[] GetBucketSnapshot(TGroupKey group) /// /// Gets number of items in requested group. /// - [MethodImpl(MethodImplOptions.Synchronized)] public int GetBucketCount(TGroupKey group) { + using var lockRelease = Lock.Acquire(); + if (group is null) throw new ArgumentNullException(nameof(group)); return _buckets.TryGetValue(group, out EnhancedSortedSet? bucket) ? bucket.Count : 0; } @@ -128,7 +135,6 @@ public int GetBucketCount(TGroupKey group) /// /// Takes first element in supplied comparer order. /// - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryTakeFirst(out TValue? first) { if (GetFirsts().Min is TValue min) @@ -140,9 +146,10 @@ public bool TryTakeFirst(out TValue? first) /// /// Returns best element of each bucket in supplied comparer order. /// - [MethodImpl(MethodImplOptions.Synchronized)] public EnhancedSortedSet GetFirsts() { + using var lockRelease = Lock.Acquire(); + EnhancedSortedSet sortedValues = new(_sortedComparer); foreach (KeyValuePair> bucket in _buckets) { @@ -172,11 +179,14 @@ protected void UpdateWorstValue() => /// Removed element or null. /// Bucket for same sender transactions. /// If element was removed. False if element was not present in pool. - [MethodImpl(MethodImplOptions.Synchronized)] - private bool TryRemove(TKey key, out TValue? value, [NotNullWhen(true)] out ICollection? bucket) => - TryRemove(key, false, out value, out bucket); + private bool TryRemove(TKey key, out TValue? value, [NotNullWhen(true)] out ICollection? bucket) + { + using var lockRelease = Lock.Acquire(); - private bool TryRemove(TKey key, bool evicted, [NotNullWhen(true)] out TValue? value, out ICollection? bucket) + return TryRemoveNonLocked(key, false, out value, out bucket); + } + + protected bool TryRemoveNonLocked(TKey key, bool evicted, [NotNullWhen(true)] out TValue? value, out ICollection? bucket) { if (_cacheMap.TryGetValue(key, out value) && value != null) { @@ -219,10 +229,8 @@ private bool TryRemove(TKey key, bool evicted, [NotNullWhen(true)] out TValue? v protected abstract TKey GetKey(TValue value); - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryRemove(TKey key, [NotNullWhen(true)] out TValue? value) => TryRemove(key, out value, out _); - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryRemove(TKey key) => TryRemove(key, out _, out _); /// @@ -231,9 +239,10 @@ private bool TryRemove(TKey key, bool evicted, [NotNullWhen(true)] out TValue? v /// Given GroupKey, which elements are checked. /// Predicated criteria. /// Elements matching predicated criteria. - [MethodImpl(MethodImplOptions.Synchronized)] public IEnumerable TakeWhile(TGroupKey groupKey, Predicate where) { + using var lockRelease = Lock.Acquire(); + if (_buckets.TryGetValue(groupKey, out EnhancedSortedSet? bucket)) { using EnhancedSortedSet.Enumerator enumerator = bucket!.GetEnumerator(); @@ -260,9 +269,10 @@ public IEnumerable TakeWhile(TGroupKey groupKey, Predicate where /// /// Key to check presence. /// True if element is present in pool. - [MethodImpl(MethodImplOptions.Synchronized)] public bool ContainsKey(TKey key) { + using var lockRelease = Lock.Acquire(); + return _cacheMap.ContainsKey(key); } @@ -272,9 +282,10 @@ public bool ContainsKey(TKey key) /// Key to be returned. /// Returned element or null. /// If element retrieval succeeded. True if element was present in pool. - [MethodImpl(MethodImplOptions.Synchronized)] public virtual bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) { + using var lockRelease = Lock.Acquire(); + return _cacheMap.TryGetValue(key, out value) && value != null; } @@ -285,9 +296,10 @@ public virtual bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) /// Element to insert. /// Element removed because of exceeding capacity /// If element was inserted. False if element was already present in pool. - [MethodImpl(MethodImplOptions.Synchronized)] public virtual bool TryInsert(TKey key, TValue value, out TValue? removed) { + using var lockRelease = Lock.Acquire(); + if (CanInsert(key, value)) { TGroupKey group = MapToGroup(value); @@ -311,7 +323,6 @@ public virtual bool TryInsert(TKey key, TValue value, out TValue? removed) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryInsert(TKey key, TValue value) => TryInsert(key, value, out _); private void RemoveLast(out TValue? removed) @@ -319,7 +330,7 @@ private void RemoveLast(out TValue? removed) TKey? key = _worstValue.GetValueOrDefault().Value; if (key is not null) { - TryRemove(key, true, out removed, out _); + TryRemoveNonLocked(key, true, out removed, out _); } else { @@ -401,13 +412,17 @@ protected virtual bool Remove(TKey key, TValue value) private void UpdateIsFull() => _isFull = _cacheMap.Count >= _capacity; - [MethodImpl(MethodImplOptions.Synchronized)] - public bool ContainsBucket(TGroupKey groupKey) => - _buckets.ContainsKey(groupKey); + public bool ContainsBucket(TGroupKey groupKey) + { + using var lockRelease = Lock.Acquire(); + + return _buckets.ContainsKey(groupKey); + } - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryGetBucket(TGroupKey groupKey, out TValue[] items) { + using var lockRelease = Lock.Acquire(); + if (_buckets.TryGetValue(groupKey, out EnhancedSortedSet? bucket)) { items = bucket.ToArray(); @@ -418,9 +433,10 @@ public bool TryGetBucket(TGroupKey groupKey, out TValue[] items) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] public bool TryGetBucketsWorstValue(TGroupKey groupKey, out TValue? item) { + using var lockRelease = Lock.Acquire(); + if (_buckets.TryGetValue(groupKey, out EnhancedSortedSet? bucket)) { item = bucket.Max; @@ -431,9 +447,10 @@ public bool TryGetBucketsWorstValue(TGroupKey groupKey, out TValue? item) return false; } - [MethodImpl(MethodImplOptions.Synchronized)] public void UpdatePool(Func, IEnumerable<(TValue Tx, Action? Change)>> changingElements) { + using var lockRelease = Lock.Acquire(); + foreach ((TGroupKey groupKey, EnhancedSortedSet bucket) in _buckets) { Debug.Assert(bucket.Count > 0); @@ -442,9 +459,10 @@ public void UpdatePool(Func, IEnumerable<( } } - [MethodImpl(MethodImplOptions.Synchronized)] public void UpdateGroup(TGroupKey groupKey, Func, IEnumerable<(TValue Tx, Action? Change)>> changingElements) { + using var lockRelease = Lock.Acquire(); + if (groupKey is null) throw new ArgumentNullException(nameof(groupKey)); if (_buckets.TryGetValue(groupKey, out EnhancedSortedSet? bucket)) { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs index a035dcd6c4d..abb9a8bd3b7 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs @@ -71,19 +71,20 @@ protected override void UpdateGroup(Address groupKey, EnhancedSortedSet, IEnumerable<(Transaction Tx, UInt256? changedGasBottleneck)>> changingElements) { + using var lockRelease = Lock.Acquire(); + foreach ((Address address, EnhancedSortedSet bucket) in _buckets) { Debug.Assert(bucket.Count > 0); Account? account = accounts.GetAccount(address); - UpdateGroup(address, account, bucket, changingElements); + UpdateGroupNonLocked(address, account, bucket, changingElements); } } - private void UpdateGroup(Address groupKey, Account groupValue, EnhancedSortedSet bucket, Func, IEnumerable<(Transaction Tx, UInt256? changedGasBottleneck)>> changingElements) + private void UpdateGroupNonLocked(Address groupKey, Account groupValue, EnhancedSortedSet bucket, Func, IEnumerable<(Transaction Tx, UInt256? changedGasBottleneck)>> changingElements) { _transactionsToRemove.Clear(); Transaction? lastElement = bucket.Max; @@ -114,25 +115,26 @@ private void UpdateGroup(Address groupKey, Account groupValue, EnhancedSortedSet ReadOnlySpan txs = CollectionsMarshal.AsSpan(_transactionsToRemove); for (int i = 0; i < txs.Length; i++) { - TryRemove(txs[i].Hash!); + TryRemoveNonLocked(txs[i].Hash!, evicted: false, out _, out _); } } - [MethodImpl(MethodImplOptions.Synchronized)] public void UpdateGroup(Address groupKey, Account groupValue, Func, IEnumerable<(Transaction Tx, UInt256? changedGasBottleneck)>> changingElements) { + using var lockRelease = Lock.Acquire(); + ArgumentNullException.ThrowIfNull(groupKey); if (_buckets.TryGetValue(groupKey, out EnhancedSortedSet? bucket)) { Debug.Assert(bucket.Count > 0); - UpdateGroup(groupKey, groupValue, bucket, changingElements); + UpdateGroupNonLocked(groupKey, groupValue, bucket, changingElements); } } public virtual void VerifyCapacity() { - if (Count > _poolCapacity && _logger.IsWarn) + if (_logger.IsWarn && Count > _poolCapacity) _logger.Warn($"TxPool exceeds the config size {Count}/{_poolCapacity}"); } } diff --git a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs index 2f2e264e3d6..a91d89e4978 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs @@ -22,7 +22,7 @@ public NotSupportedTxFilter(ITxPoolConfig txPoolConfig, ILogger logger) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) { - if (!_txPoolConfig.BlobSupportEnabled && tx.SupportsBlobs) + if (_txPoolConfig.BlobsSupport.IsDisabled() && tx.SupportsBlobs) { Metrics.PendingTransactionsNotSupportedTxType++; if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, blob transactions are not supported."); diff --git a/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs new file mode 100644 index 00000000000..7adb61d7c05 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; + +namespace Nethermind.TxPool; + +public interface IBlobTxStorage : ITxStorage +{ + bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions); + void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions); + void DeleteBlobTransactionsFromBlock(long blockNumber); +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 02420aca0b0..992c2c8db92 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -16,11 +16,16 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "2048", Description = "The max number of transactions held in the mempool (the more transactions in the mempool, the more memory used).")] int Size { get; set; } - [ConfigItem(DefaultValue = "false", Description = "Whether to enable blob transactions.")] - bool BlobSupportEnabled { get; set; } - - [ConfigItem(DefaultValue = "false", Description = "Whether to store blob transactions in the database.")] - bool PersistentBlobStorageEnabled { get; set; } + [ConfigItem( + Description = """ + Blobs support mode: + + - `Disabled`: No support for blob transactions + - `InMemory`: Blob transactions stored only in memory + - `Storage`: Blob transactions stored in db + - `StorageWithReorgs`: Blob transactions stored in db with support for restoring reorganized blob transactions to blob pool + """, DefaultValue = "Disabled")] + BlobsSupportMode BlobsSupport { get; set; } [ConfigItem(DefaultValue = "16384", Description = "The max number of full blob transactions stored in the database (increasing the number of transactions in the blob pool also results in higher memory usage). The default value uses max 13GB for 6 blobs where one blob is 2GB (16386 * 128KB).")] int PersistentBlobStorageSize { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index c4b8c933b0b..6eeaf9df353 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -10,7 +10,7 @@ namespace Nethermind.TxPool; -public class NullBlobTxStorage : ITxStorage +public class NullBlobTxStorage : IBlobTxStorage { public static NullBlobTxStorage Instance { get; } = new(); @@ -25,4 +25,14 @@ public bool TryGet(in ValueHash256 hash, Address sender, in UInt256 timestamp, [ public void Add(Transaction transaction) { } public void Delete(in ValueHash256 hash, in UInt256 timestamp) { } + + public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions) + { + blockBlobTransactions = default; + return false; + } + + public void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions) { } + + public void DeleteBlobTransactionsFromBlock(long blockNumber) { } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 701c2bf9eea..26f204f4103 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -31,8 +31,6 @@ namespace Nethermind.TxPool /// public class TxPool : ITxPool, IDisposable { - private readonly object _locker = new(); - private readonly IIncomingTxFilter[] _preHashFilters; private readonly IIncomingTxFilter[] _postHashFilters; @@ -44,8 +42,11 @@ public class TxPool : ITxPool, IDisposable private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; + private readonly IEthereumEcdsa _ecdsa; + private readonly IBlobTxStorage _blobTxStorage; private readonly IChainHeadInfoProvider _headInfo; private readonly ITxPoolConfig _txPoolConfig; + private readonly bool _blobReorgsSupportEnabled; private readonly ILogger _logger; @@ -77,7 +78,7 @@ public class TxPool : ITxPool, IDisposable /// /// public TxPool(IEthereumEcdsa ecdsa, - ITxStorage blobTxStorage, + IBlobTxStorage blobTxStorage, IChainHeadInfoProvider chainHeadInfoProvider, ITxPoolConfig txPoolConfig, ITxValidator validator, @@ -88,8 +89,11 @@ public TxPool(IEthereumEcdsa ecdsa, bool thereIsPriorityContract = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _ecdsa = ecdsa ?? throw new ArgumentNullException(nameof(ecdsa)); + _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; + _blobReorgsSupportEnabled = txPoolConfig.BlobsSupport.SupportsReorgs(); _accounts = _headInfo.AccountStateProvider; _specProvider = _headInfo.SpecProvider; @@ -102,9 +106,9 @@ public TxPool(IEthereumEcdsa ecdsa, _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); - _blobTransactions = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true } + _blobTransactions = txPoolConfig.BlobsSupport.IsPersistentStorage() ? new PersistentBlobTxDistinctSortedPool(blobTxStorage, _txPoolConfig, comparer, logManager) - : new BlobTxDistinctSortedPool(txPoolConfig.BlobSupportEnabled ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); + : new BlobTxDistinctSortedPool(txPoolConfig.BlobsSupport == BlobsSupportMode.InMemory ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); if (_blobTransactions.Count > 0) _blobTransactions.UpdatePool(_accounts, _updateBucket); _headInfo.HeadChanged += OnHeadChange; @@ -199,7 +203,7 @@ private void ProcessNewHeads() try { ReAddReorganisedTransactions(args.PreviousBlock); - RemoveProcessedTransactions(args.Block.Transactions); + RemoveProcessedTransactions(args.Block); UpdateBuckets(); _broadcaster.OnNewHead(); Metrics.TransactionCount = _transactions.Count; @@ -236,11 +240,29 @@ private void ReAddReorganisedTransactions(Block? previousBlock) _hashCache.Delete(tx.Hash!); SubmitTx(tx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); } + + if (_blobReorgsSupportEnabled + && _blobTxStorage.TryGetBlobTransactionsFromBlock(previousBlock.Number, out Transaction[]? blobTxs) + && blobTxs is not null) + { + foreach (Transaction blobTx in blobTxs) + { + if (_logger.IsTrace) _logger.Trace($"Readded tx {blobTx.Hash} from reorged block {previousBlock.Number} (hash {previousBlock.Hash}) to blob pool"); + _hashCache.Delete(blobTx.Hash!); + blobTx.SenderAddress ??= _ecdsa.RecoverAddress(blobTx); + SubmitTx(blobTx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); + } + if (_logger.IsDebug) _logger.Debug($"Readded txs from reorged block {previousBlock.Number} (hash {previousBlock.Hash}) to blob pool"); + + _blobTxStorage.DeleteBlobTransactionsFromBlock(previousBlock.Number); + } } } - private void RemoveProcessedTransactions(Transaction[] blockTransactions) + private void RemoveProcessedTransactions(Block block) { + Transaction[] blockTransactions = block.Transactions; + using ArrayPoolList blobTxsToSave = new(Eip4844Constants.GetMaxBlobsPerBlock()); long discoveredForPendingTxs = 0; long discoveredForHashCache = 0; long eip1559Txs = 0; @@ -249,31 +271,46 @@ private void RemoveProcessedTransactions(Transaction[] blockTransactions) for (int i = 0; i < blockTransactions.Length; i++) { - Transaction transaction = blockTransactions[i]; - Hash256 txHash = transaction.Hash ?? throw new ArgumentException("Hash was unexpectedly null!"); + Transaction blockTx = blockTransactions[i]; + Hash256 txHash = blockTx.Hash ?? throw new ArgumentException("Hash was unexpectedly null!"); - if (!IsKnown(txHash)) + if (blockTx.Supports1559) { - discoveredForHashCache++; + eip1559Txs++; } - if (!RemoveIncludedTransaction(transaction)) + if (blockTx.SupportsBlobs) { - discoveredForPendingTxs++; + blobTxs++; + blobs += blockTx.BlobVersionedHashes?.Length ?? 0; + + if (_blobReorgsSupportEnabled) + { + if (_blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) + { + if (_logger.IsTrace) _logger.Trace($"Saved processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db"); + blobTxsToSave.Add(fullBlobTx); + } + else if (_logger.IsTrace) _logger.Trace($"Skipped adding processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db - not found in blob pool"); + } } - if (transaction.Supports1559) + if (!IsKnown(txHash)) { - eip1559Txs++; + discoveredForHashCache++; } - if (transaction.SupportsBlobs) + if (!RemoveIncludedTransaction(blockTx)) { - blobTxs++; - blobs += transaction.BlobVersionedHashes?.Length ?? 0; + discoveredForPendingTxs++; } } + if (blobTxsToSave.Count > 0) + { + _blobTxStorage.AddBlobTransactionsFromBlock(block.Number, blobTxsToSave); + } + long transactionsInBlock = blockTransactions.Length; if (transactionsInBlock != 0) { @@ -391,54 +428,51 @@ private AcceptTxResult FilterTransactions(Transaction tx, TxHandlingOptions hand private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPersistentBroadcast) { - lock (_locker) - { - bool eip1559Enabled = _specProvider.GetCurrentHeadSpec().IsEip1559Enabled; - UInt256 effectiveGasPrice = tx.CalculateEffectiveGasPrice(eip1559Enabled, _headInfo.CurrentBaseFee); - TxDistinctSortedPool relevantPool = (tx.SupportsBlobs ? _blobTransactions : _transactions); + bool eip1559Enabled = _specProvider.GetCurrentHeadSpec().IsEip1559Enabled; + UInt256 effectiveGasPrice = tx.CalculateEffectiveGasPrice(eip1559Enabled, _headInfo.CurrentBaseFee); + TxDistinctSortedPool relevantPool = (tx.SupportsBlobs ? _blobTransactions : _transactions); - relevantPool.TryGetBucketsWorstValue(tx.SenderAddress!, out Transaction? worstTx); - tx.GasBottleneck = (worstTx is null || effectiveGasPrice <= worstTx.GasBottleneck) - ? effectiveGasPrice - : worstTx.GasBottleneck; + relevantPool.TryGetBucketsWorstValue(tx.SenderAddress!, out Transaction? worstTx); + tx.GasBottleneck = (worstTx is null || effectiveGasPrice <= worstTx.GasBottleneck) + ? effectiveGasPrice + : worstTx.GasBottleneck; - bool inserted = relevantPool.TryInsert(tx.Hash!, tx, out Transaction? removed); + bool inserted = relevantPool.TryInsert(tx.Hash!, tx, out Transaction? removed); - if (!inserted) - { - // it means it failed on adding to the pool - it is possible when new tx has the same sender - // and nonce as already existent tx and is not good enough to replace it - Metrics.PendingTransactionsPassedFiltersButCannotReplace++; - return AcceptTxResult.ReplacementNotAllowed; - } + if (!inserted) + { + // it means it failed on adding to the pool - it is possible when new tx has the same sender + // and nonce as already existent tx and is not good enough to replace it + Metrics.PendingTransactionsPassedFiltersButCannotReplace++; + return AcceptTxResult.ReplacementNotAllowed; + } - if (tx.Hash == removed?.Hash) + if (tx.Hash == removed?.Hash) + { + // it means it was added and immediately evicted - pool was full of better txs + if (isPersistentBroadcast) { - // it means it was added and immediately evicted - pool was full of better txs - if (isPersistentBroadcast) - { - // we are adding only to persistent broadcast - not good enough for standard pool, - // but can be good enough for TxBroadcaster pool - for local txs only - _broadcaster.Broadcast(tx, isPersistentBroadcast); - } - Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees++; - return AcceptTxResult.FeeTooLowToCompete; + // we are adding only to persistent broadcast - not good enough for standard pool, + // but can be good enough for TxBroadcaster pool - for local txs only + _broadcaster.Broadcast(tx, isPersistentBroadcast); } + Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees++; + return AcceptTxResult.FeeTooLowToCompete; + } - relevantPool.UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); - Metrics.PendingTransactionsAdded++; - if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } - if (tx.SupportsBlobs) { Metrics.PendingBlobTransactionsAdded++; } + relevantPool.UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); + Metrics.PendingTransactionsAdded++; + if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } + if (tx.SupportsBlobs) { Metrics.PendingBlobTransactionsAdded++; } - if (removed is not null) - { - EvictedPending?.Invoke(this, new TxEventArgs(removed)); - // transaction which was on last position in sorted TxPool and was deleted to give - // a place for a newly added tx (with higher priority) is now removed from hashCache - // to give it opportunity to come back to TxPool in the future, when fees drops - _hashCache.DeleteFromLongTerm(removed.Hash!); - Metrics.PendingTransactionsEvicted++; - } + if (removed is not null) + { + EvictedPending?.Invoke(this, new TxEventArgs(removed)); + // transaction which was on last position in sorted TxPool and was deleted to give + // a place for a newly added tx (with higher priority) is now removed from hashCache + // to give it opportunity to come back to TxPool in the future, when fees drops + _hashCache.DeleteFromLongTerm(removed.Hash!); + Metrics.PendingTransactionsEvicted++; } _broadcaster.Broadcast(tx, isPersistentBroadcast); @@ -519,14 +553,11 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe private void UpdateBuckets() { - lock (_locker) - { - _transactions.VerifyCapacity(); - _transactions.UpdatePool(_accounts, _updateBucket); + _transactions.VerifyCapacity(); + _transactions.UpdatePool(_accounts, _updateBucket); - _blobTransactions.VerifyCapacity(); - _blobTransactions.UpdatePool(_accounts, _updateBucket); - } + _blobTransactions.VerifyCapacity(); + _blobTransactions.UpdatePool(_accounts, _updateBucket); } private IEnumerable<(Transaction Tx, UInt256? changedGasBottleneck)> UpdateBucket(Address address, Account account, EnhancedSortedSet transactions) @@ -589,22 +620,21 @@ public bool RemoveTransaction(Hash256? hash) return false; } - bool hasBeenRemoved; - lock (_locker) - { - hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction) + bool hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction) || _blobTransactions.TryRemove(hash, out transaction); - if (transaction is null || !hasBeenRemoved) - return false; - if (hasBeenRemoved) - { - RemovedPending?.Invoke(this, new TxEventArgs(transaction)); - } + if (transaction is null || !hasBeenRemoved) + { + return false; + } - _broadcaster.StopBroadcast(hash); + if (hasBeenRemoved) + { + RemovedPending?.Invoke(this, new TxEventArgs(transaction)); } + _broadcaster.StopBroadcast(hash); + if (_logger.IsTrace) _logger.Trace($"Removed a transaction: {hash}"); return hasBeenRemoved; @@ -616,20 +646,14 @@ public bool ContainsTx(Hash256 hash, TxType txType) => txType == TxType.Blob public bool TryGetPendingTransaction(Hash256 hash, out Transaction? transaction) { - lock (_locker) - { - return _transactions.TryGetValue(hash, out transaction) - || _blobTransactions.TryGetValue(hash, out transaction) - || _broadcaster.TryGetPersistentTx(hash, out transaction); - } + return _transactions.TryGetValue(hash, out transaction) + || _blobTransactions.TryGetValue(hash, out transaction) + || _broadcaster.TryGetPersistentTx(hash, out transaction); } public bool TryGetPendingBlobTransaction(Hash256 hash, [NotNullWhen(true)] out Transaction? blobTransaction) { - lock (_locker) - { - return _blobTransactions.TryGetValue(hash, out blobTransaction); - } + return _blobTransactions.TryGetValue(hash, out blobTransaction); } // only for tests - to test sorting diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 7ae41d8bebb..b2484b33799 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -8,8 +8,7 @@ public class TxPoolConfig : ITxPoolConfig public int PeerNotificationThreshold { get; set; } = 5; public int MinBaseFeeThreshold { get; set; } = 70; public int Size { get; set; } = 2048; - public bool BlobSupportEnabled { get; set; } = false; - public bool PersistentBlobStorageEnabled { get; set; } = false; + public BlobsSupportMode BlobsSupport { get; set; } = BlobsSupportMode.Disabled; public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 13GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); // practical max - something between, but closer to 2GB than 12GB. Geth is limiting it to 10GB. // every day about 21600 blobs will be included (7200 blocks per day * 3 blob target) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index d5d51c8ce4f..9c6bff122fe 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -1,12 +1,15 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Text.Json; using Nethermind.Tools.Kute.Extensions; using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.JsonRpcMethodFilter; using Nethermind.Tools.Kute.JsonRpcSubmitter; +using Nethermind.Tools.Kute.JsonRpcValidator; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; +using Nethermind.Tools.Kute.ResponseTracer; namespace Nethermind.Tools.Kute; @@ -16,6 +19,8 @@ class Application private readonly IMessageProvider _msgProvider; private readonly IJsonRpcSubmitter _submitter; + private readonly IJsonRpcValidator _validator; + private readonly IResponseTracer _responseTracer; private readonly IProgressReporter _progressReporter; private readonly IMetricsConsumer _metricsConsumer; private readonly IJsonRpcMethodFilter _methodFilter; @@ -23,6 +28,8 @@ class Application public Application( IMessageProvider msgProvider, IJsonRpcSubmitter submitter, + IJsonRpcValidator validator, + IResponseTracer responseTracer, IProgressReporter progressReporter, IMetricsConsumer metricsConsumer, IJsonRpcMethodFilter methodFilter @@ -30,6 +37,8 @@ IJsonRpcMethodFilter methodFilter { _msgProvider = msgProvider; _submitter = submitter; + _validator = validator; + _responseTracer = responseTracer; _progressReporter = progressReporter; _metricsConsumer = metricsConsumer; _methodFilter = methodFilter; @@ -43,24 +52,41 @@ public async Task Run() { await foreach (var (jsonRpc, n) in _msgProvider.Messages.Indexed(startingFrom: 1)) { + _progressReporter.ReportProgress(n); + _metrics.TickMessages(); switch (jsonRpc) { case null: + { _metrics.TickFailed(); break; - + } case JsonRpc.BatchJsonRpc batch: + { + JsonDocument? result; using (_metrics.TimeBatch()) { - await _submitter.Submit(batch); + result = await _submitter.Submit(batch); } - break; + if (_validator.IsInvalid(batch, result)) + { + _metrics.TickFailed(); + } + else + { + _metrics.TickSucceeded(); + } + await _responseTracer.TraceResponse(result); + + break; + } case JsonRpc.SingleJsonRpc single: + { if (single.IsResponse) { _metrics.TickResponses(); @@ -79,18 +105,30 @@ public async Task Run() continue; } + JsonDocument? result; using (_metrics.TimeMethod(single.MethodName)) { - await _submitter.Submit(single); + result = await _submitter.Submit(single); } - break; + if (_validator.IsInvalid(single, result)) + { + _metrics.TickFailed(); + } + else + { + _metrics.TickSucceeded(); + } + + await _responseTracer.TraceResponse(result); + break; + } default: + { throw new ArgumentOutOfRangeException(nameof(jsonRpc)); + } } - - _progressReporter.ReportProgress(n); } } diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index 87c0876ac21..b2bd10b2881 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -21,7 +21,7 @@ public class Config longName: "address", Required = false, Default = "http://localhost:8551", - HelpText = "Address where to send JSON RPC calls" + HelpText = "Address where to send JSON RPC requests" )] public string HostAddress { get; } @@ -29,7 +29,7 @@ public class Config shortName: 's', longName: "secret", Required = true, - HelpText = "Path to file with hex encoded secret for JWT authentication" + HelpText = "Path to File with hex encoded secret for JWT authentication" )] public string JwtSecretFilePath { get; } @@ -75,10 +75,19 @@ public class Config Separator = ',', Required = false, Default = new string[] { }, - HelpText = "A comma separated List of regexes of methods to be executed" + HelpText = "A comma separated List of regexes of methods to be executed with optional limits" )] public IEnumerable MethodFilters { get; } + [Option( + shortName: 'r', + longName: "responses", + Required = false, + Default = null, + HelpText = "Path to File to store JSON-RPC responses" + )] + public string? ResponsesTraceFile { get; } + public Config( string messagesFilePath, string hostAddress, @@ -87,7 +96,8 @@ public Config( bool dryRun, bool showProgress, MetricsOutputFormatter metricsOutputFormatter, - IEnumerable methodFilters + IEnumerable methodFilters, + string? responsesTraceFile ) { MessagesFilePath = messagesFilePath; @@ -98,6 +108,6 @@ IEnumerable methodFilters ShowProgress = showProgress; MetricsOutputFormatter = metricsOutputFormatter; MethodFilters = methodFilters; - ShowProgress = showProgress; + ResponsesTraceFile = responsesTraceFile; } } diff --git a/tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs b/tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs new file mode 100644 index 00000000000..b15ef8125bc --- /dev/null +++ b/tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using App.Metrics; +using App.Metrics.ReservoirSampling; +using App.Metrics.ReservoirSampling.Uniform; + +namespace Nethermind.Tools.Kute.Extensions; + +public class CompleteReservoir : IReservoir +{ + private const int DefaultSize = 10_000; + + private readonly List _values; + + public CompleteReservoir() : this(DefaultSize) { } + + public CompleteReservoir(int size) + { + _values = new List(size); + } + + public IReservoirSnapshot GetSnapshot(bool resetReservoir) + { + long count = _values.Count; + double sum = _values.Sum(v => v.Value); + IEnumerable values = _values.Select(v => v.Value); + + if (resetReservoir) + { + _values.Clear(); + } + + return new UniformSnapshot(count, sum, values); + } + + public IReservoirSnapshot GetSnapshot() => GetSnapshot(false); + + public void Reset() => _values.Clear(); + + public void Update(long value, string userValue) => _values.Add(new UserValueWrapper(value, userValue)); + + public void Update(long value) => _values.Add(new UserValueWrapper(value)); + +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs new file mode 100644 index 00000000000..78e4824ea41 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Tools.Kute.JsonRpcMethodFilter; + +class LimitedJsonRpcMethodFilter : IJsonRpcMethodFilter +{ + private readonly IJsonRpcMethodFilter _filter; + + private int _usagesLeft; + + public LimitedJsonRpcMethodFilter(IJsonRpcMethodFilter filter, int limit) + { + _filter = filter; + _usagesLeft = limit; + } + + public bool ShouldSubmit(string methodName) + { + if (_filter.ShouldSubmit(methodName)) + { + if (_usagesLeft == 0) + { + return false; + } + + _usagesLeft--; + return true; + } + + return false; + } +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs index 324301214a8..a628b3e9245 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs @@ -1,18 +1,25 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Text.RegularExpressions; - namespace Nethermind.Tools.Kute.JsonRpcMethodFilter; -class PatternJsonRpcMethodFilter : IJsonRpcMethodFilter +public class PatternJsonRpcMethodFilter : IJsonRpcMethodFilter { - private readonly Regex _pattern; + private const char PatternSeparator = '='; + private readonly IJsonRpcMethodFilter _filter; public PatternJsonRpcMethodFilter(string pattern) { - _pattern = new Regex(pattern); + var splitted = pattern.Split(PatternSeparator); + + var regex = new RegexJsonRpcMethodFilter(splitted[0]); + _filter = splitted.Length switch + { + 1 => regex, + 2 => new LimitedJsonRpcMethodFilter(regex, int.Parse(splitted[1])), + _ => throw new ArgumentException($"Unexpected pattern: {pattern}"), + }; } - public bool ShouldSubmit(string methodName) => _pattern.IsMatch(methodName); + public bool ShouldSubmit(string methodName) => _filter.ShouldSubmit(methodName); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs new file mode 100644 index 00000000000..07fd79ec4ad --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.RegularExpressions; + +namespace Nethermind.Tools.Kute.JsonRpcMethodFilter; + +class RegexJsonRpcMethodFilter : IJsonRpcMethodFilter +{ + private readonly Regex _pattern; + + public RegexJsonRpcMethodFilter(string pattern) + { + _pattern = new Regex(pattern); + } + + public bool ShouldSubmit(string methodName) => _pattern.IsMatch(methodName); +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs index 9fd1774ac3f..56a28e51168 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs @@ -5,6 +5,7 @@ using System.Net.Http.Headers; using System.Net.Mime; using System.Text; +using System.Text.Json; using Nethermind.Tools.Kute.Auth; namespace Nethermind.Tools.Kute.JsonRpcSubmitter; @@ -22,7 +23,7 @@ public HttpJsonRpcSubmitter(HttpClient httpClient, IAuth auth, string hostAddres _uri = new Uri(hostAddress); } - public async Task Submit(JsonRpc rpc) + public async Task Submit(JsonRpc rpc) { var request = new HttpRequestMessage(HttpMethod.Post, _uri) { @@ -32,7 +33,9 @@ public async Task Submit(JsonRpc rpc) var response = await _httpClient.SendAsync(request); if (response.StatusCode != HttpStatusCode.OK) { - throw new HttpRequestException($"Expected {HttpStatusCode.OK}, got {response.StatusCode}"); + return null; } + var content = await response.Content.ReadAsStreamAsync(); + return await JsonSerializer.DeserializeAsync(content); } } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs index bb9f5038829..5d2afe74df0 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Text.Json; + namespace Nethermind.Tools.Kute.JsonRpcSubmitter; interface IJsonRpcSubmitter { - Task Submit(JsonRpc rpc); + Task Submit(JsonRpc rpc); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs index d38cc4d410f..18e0ce01425 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs @@ -1,22 +1,12 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using Nethermind.Tools.Kute.Auth; +using System.Text.Json; namespace Nethermind.Tools.Kute.JsonRpcSubmitter; class NullJsonRpcSubmitter : IJsonRpcSubmitter { - private readonly IAuth _auth; - public NullJsonRpcSubmitter(IAuth auth) - { - _auth = auth; - } - - public Task Submit(JsonRpc rpc) - { - _ = _auth.AuthToken; - return Task.CompletedTask; - } + public Task Submit(JsonRpc rpc) => Task.FromResult(null); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs new file mode 100644 index 00000000000..11d935cbc8a --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public class ComposedJsonRpcValidator : IJsonRpcValidator +{ + private readonly IEnumerable _validators; + + public ComposedJsonRpcValidator(IEnumerable validators) + { + _validators = validators; + } + + public bool IsValid(JsonRpc request, JsonDocument? document) => _validators.All(validator => validator.IsValid(request, document)); +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs new file mode 100644 index 00000000000..dc302584065 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace Nethermind.Tools.Kute.JsonRpcValidator.Eth; + +public class NewPayloadJsonRpcValidator : IJsonRpcValidator +{ + private readonly Regex _pattern = new Regex("engine_newPayload"); + + public bool IsValid(JsonRpc request, JsonDocument? response) + { + // If preconditions are not met, then mark it as Valid. + if (!ShouldValidateRequest(request) || response is null) + { + return true; + } + + if (!response.RootElement.TryGetProperty("result", out var result)) + { + return false; + } + + if (!result.TryGetProperty("status", out var status)) + { + return false; + } + + return status.GetString() == "VALID"; + } + + private bool ShouldValidateRequest(JsonRpc request) + { + if (request is JsonRpc.SingleJsonRpc { MethodName: not null } single) + { + if (_pattern.IsMatch(single.MethodName)) + { + return true; + } + } + + return false; + } +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs new file mode 100644 index 00000000000..e2af545fee1 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator.Eth; + +public class NonErrorJsonRpcValidator : IJsonRpcValidator +{ + public bool IsValid(JsonRpc request, JsonDocument? response) + { + if (response is null) + { + return false; + } + + return !response.RootElement.TryGetProperty("error", out _); + } +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs new file mode 100644 index 00000000000..20d30f714cc --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public interface IJsonRpcValidator +{ + bool IsValid(JsonRpc request, JsonDocument? response); + bool IsInvalid(JsonRpc request, JsonDocument? response) => !IsValid(request, response); +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs new file mode 100644 index 00000000000..12ab7833a45 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public class NullJsonRpcValidator : IJsonRpcValidator +{ + public bool IsValid(JsonRpc request, JsonDocument? response) => true; +} diff --git a/tools/Nethermind.Tools.Kute/Metrics.cs b/tools/Nethermind.Tools.Kute/Metrics.cs index f3038506d8b..6c13a7e9cac 100644 --- a/tools/Nethermind.Tools.Kute/Metrics.cs +++ b/tools/Nethermind.Tools.Kute/Metrics.cs @@ -4,6 +4,7 @@ using App.Metrics; using App.Metrics.Counter; using App.Metrics.Timer; +using Nethermind.Tools.Kute.Extensions; namespace Nethermind.Tools.Kute; @@ -23,6 +24,10 @@ public class Metrics { Name = "Failed", MeasurementUnit = Unit.Items, }; + private readonly CounterOptions _succeeded = new() + { + Name = "Succeeded", MeasurementUnit = Unit.Items, + }; private readonly CounterOptions _ignoredRequests = new() { Name = "Ignored Requests", MeasurementUnit = Unit.Items @@ -39,13 +44,16 @@ public class Metrics public Metrics() { - _metrics = new MetricsBuilder().Build(); + _metrics = new MetricsBuilder() + .SampleWith.Reservoir() + .Build(); } public MetricsDataValueSource Snapshot => _metrics.Snapshot.Get(); public void TickMessages() => _metrics.Measure.Counter.Increment(_messages); public void TickFailed() => _metrics.Measure.Counter.Increment(_failed); + public void TickSucceeded() => _metrics.Measure.Counter.Increment(_succeeded); public void TickIgnoredRequests() => _metrics.Measure.Counter.Increment(_ignoredRequests); public void TickResponses() => _metrics.Measure.Counter.Increment(_responses); diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index 61f7f37b92f..79932b07db9 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -6,9 +6,12 @@ using Nethermind.Tools.Kute.Auth; using Nethermind.Tools.Kute.JsonRpcMethodFilter; using Nethermind.Tools.Kute.JsonRpcSubmitter; +using Nethermind.Tools.Kute.JsonRpcValidator; +using Nethermind.Tools.Kute.JsonRpcValidator.Eth; using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; +using Nethermind.Tools.Kute.ResponseTracer; using Nethermind.Tools.Kute.SecretProvider; using Nethermind.Tools.Kute.SystemClock; @@ -47,18 +50,43 @@ static IServiceProvider BuildServiceProvider(Config config) ); collection.AddSingleton>(new FileMessageProvider(config.MessagesFilePath)); collection.AddSingleton, JsonRpcMessageProvider>(); + collection.AddSingleton( + config.DryRun + ? new NullJsonRpcValidator() + : new ComposedJsonRpcValidator(new List + { + new NonErrorJsonRpcValidator(), new NewPayloadJsonRpcValidator(), + }) + ); collection.AddSingleton( - new ComposedJsonRpcMethodFilter(config.MethodFilters.Select(pattern => - new PatternJsonRpcMethodFilter(pattern))) + new ComposedJsonRpcMethodFilter( + config.MethodFilters + .Select(pattern => new PatternJsonRpcMethodFilter(pattern)) + .ToList() + ) ); collection.AddSingleton(provider => - config.DryRun - ? new NullJsonRpcSubmitter(provider.GetRequiredService()) - : new HttpJsonRpcSubmitter( + { + if (!config.DryRun) + { + return new HttpJsonRpcSubmitter( provider.GetRequiredService(), provider.GetRequiredService(), config.HostAddress - )); + ); + } + + // For dry runs we still want to trigger the generation of an AuthToken + // This is to ensure that all parameters required for the generation are correct, + // and not require a real run to verify that this is the case. + string _ = provider.GetRequiredService().AuthToken; + return new NullJsonRpcSubmitter(); + }); + collection.AddSingleton( + config is { DryRun: false, ResponsesTraceFile: not null } + ? new FileResponseTracer(config.ResponsesTraceFile) + : new NullResponseTracer() + ); collection.AddSingleton(provider => { if (config.ShowProgress) @@ -77,12 +105,14 @@ static IServiceProvider BuildServiceProvider(Config config) return new NullProgressReporter(); }); collection.AddSingleton(); - collection.AddSingleton(_ => config.MetricsOutputFormatter switch - { - MetricsOutputFormatter.Report => new MetricsTextOutputFormatter(), - MetricsOutputFormatter.Json => new MetricsJsonOutputFormatter(), - _ => throw new ArgumentOutOfRangeException(), - }); + collection.AddSingleton( + config.MetricsOutputFormatter switch + { + MetricsOutputFormatter.Report => new MetricsTextOutputFormatter(), + MetricsOutputFormatter.Json => new MetricsJsonOutputFormatter(), + _ => throw new ArgumentOutOfRangeException(), + } + ); return collection.BuildServiceProvider(); } diff --git a/tools/Nethermind.Tools.Kute/README.md b/tools/Nethermind.Tools.Kute/README.md index 68c3937981b..891bbdaaac7 100644 --- a/tools/Nethermind.Tools.Kute/README.md +++ b/tools/Nethermind.Tools.Kute/README.md @@ -31,13 +31,31 @@ Some typical usages are as follow: ### Use a single messages file and emit results as JSON ``` --i /rpc-logs -s keystore/jwt-secret -o Json +-i /rpc.0 -s keystore/jwt-secret -o Json ``` -### Use a single message file, using only `engine_*` and `eth_*` methods +### Use a single messages file and record all responses into a new file ``` --i /rpc.0 -s keystore/jwt-secret -f engine_*, eth_* +-i /rpc.0 -s keystore/jwt-secret -r rpc.responses.txt +``` + +### Use a single message file, using only `engine` and `eth` methods + +``` +-i /rpc.0 -s keystore/jwt-secret -f engine, eth +``` + +### Use a single message file, using only the first 100 methods + +``` +-i /rpc.0 -s keystore/jwt-secret -f .*=100 +``` + +### Use a single message file, using only the first 50 `engine_newPayloadV2` or `engine_newPayloadV3` methods + +``` +-i /rpc.0 -s keystore/jwt-secret -f engine_newPayloadV[23]=50 ``` ### Connect to a Nethermind Client running in a specific address and TTL diff --git a/tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs b/tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs new file mode 100644 index 00000000000..cd0a163ed29 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.ResponseTracer; + +public class FileResponseTracer : IResponseTracer +{ + private readonly string _tracesFilePath; + + public FileResponseTracer(string tracesFilePath) + { + _tracesFilePath = tracesFilePath; + } + + public async Task TraceResponse(JsonDocument? response) + { + await using StreamWriter sw = File.Exists(_tracesFilePath) + ? File.AppendText(_tracesFilePath) + : File.CreateText(_tracesFilePath); + + await sw.WriteLineAsync(response?.RootElement.ToString() ?? "null"); + } +} diff --git a/tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs b/tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs new file mode 100644 index 00000000000..f57bd7f68a8 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.ResponseTracer; + +public interface IResponseTracer +{ + Task TraceResponse(JsonDocument? response); +} diff --git a/tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs b/tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs new file mode 100644 index 00000000000..6440e71612a --- /dev/null +++ b/tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.ResponseTracer; + +public class NullResponseTracer : IResponseTracer +{ + public Task TraceResponse(JsonDocument? response) => Task.CompletedTask; +}