From ce49aa4514846d5ad072defc734dff029b18ad90 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 20 Jul 2023 15:30:17 +0200 Subject: [PATCH 01/41] add column for processed txs --- src/Nethermind/Nethermind.Db/BlobTxsColumns.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs index 56b4f1d4848..107372461dd 100644 --- a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs +++ b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs @@ -6,5 +6,6 @@ namespace Nethermind.Db; public enum BlobTxsColumns { FullBlobTxs, - LightBlobTxs + LightBlobTxs, + ProcessedTxs } From 3290acc6e4fe165419f6c1d5ccac8ae682cf3529 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 31 Oct 2023 14:20:44 +0100 Subject: [PATCH 02/41] add feature of returning keys only --- .../Nethermind.Db.Rocks/ColumnDb.cs | 6 +++ .../Nethermind.Db.Rocks/DbOnTheRocks.cs | 53 +++++++++++++++++++ src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs | 2 + src/Nethermind/Nethermind.Db/CompressingDb.cs | 3 ++ .../FullPruning/FullPruningDb.cs | 2 + src/Nethermind/Nethermind.Db/IDb.cs | 1 + src/Nethermind/Nethermind.Db/MemDb.cs | 2 + src/Nethermind/Nethermind.Db/NullDb.cs | 1 + src/Nethermind/Nethermind.Db/ReadOnlyDb.cs | 2 + .../Nethermind.Db/SimpleFilePublicKeyDb.cs | 2 + 10 files changed, 74 insertions(+) diff --git a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs index e5531b17c6e..21ea16497d9 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/ColumnDb.cs @@ -64,6 +64,12 @@ public void PutSpan(ReadOnlySpan key, ReadOnlySpan value, WriteFlags return _mainDb.GetAllCore(iterator); } + public IEnumerable GetAllKeys(bool ordered = false) + { + Iterator iterator = _mainDb.CreateIterator(ordered, _columnFamily); + return _mainDb.GetAllKeysCore(iterator); + } + public IEnumerable GetAllValues(bool ordered = false) { Iterator iterator = _mainDb.CreateIterator(ordered, _columnFamily); diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index b9353a140c4..82a470c73c7 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -658,6 +658,17 @@ protected internal Iterator CreateIterator(bool ordered = false, ColumnFamilyHan } } + public IEnumerable GetAllKeys(bool ordered = false) + { + if (_isDisposing) + { + throw new ObjectDisposedException($"Attempted to read form a disposed database {Name}"); + } + + Iterator iterator = CreateIterator(ordered); + return GetAllKeysCore(iterator); + } + public IEnumerable GetAllValues(bool ordered = false) { if (_isDisposing) @@ -711,6 +722,48 @@ internal IEnumerable GetAllValuesCore(Iterator iterator) } } + internal IEnumerable GetAllKeysCore(Iterator iterator) + { + try + { + try + { + iterator.SeekToFirst(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + + while (iterator.Valid()) + { + yield return iterator.Key(); + try + { + iterator.Next(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + } + finally + { + try + { + iterator.Dispose(); + } + catch (RocksDbSharpException e) + { + CreateMarkerIfCorrupt(e); + throw; + } + } + } + public IEnumerable> GetAllCore(Iterator iterator) { try diff --git a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs index cc1309f755f..c9adaa5b6a8 100644 --- a/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs +++ b/src/Nethermind/Nethermind.Db.Rpc/RpcDb.cs @@ -77,6 +77,8 @@ public void Clear() { } public IEnumerable> GetAll(bool ordered = false) => _recordDb.GetAll(); + public IEnumerable GetAllKeys(bool ordered = false) => _recordDb.GetAllKeys(); + public IEnumerable GetAllValues(bool ordered = false) => _recordDb.GetAllValues(); public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/CompressingDb.cs b/src/Nethermind/Nethermind.Db/CompressingDb.cs index 8a2dc1d26b9..43bd72d3e98 100644 --- a/src/Nethermind/Nethermind.Db/CompressingDb.cs +++ b/src/Nethermind/Nethermind.Db/CompressingDb.cs @@ -127,6 +127,9 @@ private static ReadOnlySpan Compress(ReadOnlySpan bytes, Span public IEnumerable> GetAll(bool ordered = false) => _wrapped.GetAll(ordered) .Select(kvp => new KeyValuePair(kvp.Key, Decompress(kvp.Value))); + public IEnumerable GetAllKeys(bool ordered = false) => + _wrapped.GetAllKeys(ordered).Select(Decompress); + public IEnumerable GetAllValues(bool ordered = false) => _wrapped.GetAllValues(ordered).Select(Decompress); diff --git a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs index 9175ba9b9e9..33f94731627 100755 --- a/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs +++ b/src/Nethermind/Nethermind.Db/FullPruning/FullPruningDb.cs @@ -126,6 +126,8 @@ public void Dispose() public IEnumerable> GetAll(bool ordered = false) => _currentDb.GetAll(ordered); + public IEnumerable GetAllKeys(bool ordered = false) => _currentDb.GetAllKeys(ordered); + public IEnumerable GetAllValues(bool ordered = false) => _currentDb.GetAllValues(ordered); // we need to remove from both DB's diff --git a/src/Nethermind/Nethermind.Db/IDb.cs b/src/Nethermind/Nethermind.Db/IDb.cs index d5d0cab97c2..1eb416712ca 100644 --- a/src/Nethermind/Nethermind.Db/IDb.cs +++ b/src/Nethermind/Nethermind.Db/IDb.cs @@ -12,6 +12,7 @@ public interface IDb : IKeyValueStoreWithBatching, IDbMeta, IDisposable string Name { get; } KeyValuePair[] this[byte[][] keys] { get; } IEnumerable> GetAll(bool ordered = false); + IEnumerable GetAllKeys(bool ordered = false); IEnumerable GetAllValues(bool ordered = false); public IReadOnlyDb CreateReadOnly(bool createInMemWriteStore) => new ReadOnlyDb(this, createInMemWriteStore); diff --git a/src/Nethermind/Nethermind.Db/MemDb.cs b/src/Nethermind/Nethermind.Db/MemDb.cs index 7572adf7733..24a9c820255 100644 --- a/src/Nethermind/Nethermind.Db/MemDb.cs +++ b/src/Nethermind/Nethermind.Db/MemDb.cs @@ -88,6 +88,8 @@ public void Clear() public IEnumerable> GetAll(bool ordered = false) => _db; + public IEnumerable GetAllKeys(bool ordered = false) => Keys; + public IEnumerable GetAllValues(bool ordered = false) => Values; public virtual IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/NullDb.cs b/src/Nethermind/Nethermind.Db/NullDb.cs index 0d84f04b734..b914c537b5a 100644 --- a/src/Nethermind/Nethermind.Db/NullDb.cs +++ b/src/Nethermind/Nethermind.Db/NullDb.cs @@ -54,6 +54,7 @@ public void Clear() { } public IEnumerable> GetAll(bool ordered = false) => Enumerable.Empty>(); + public IEnumerable GetAllKeys(bool ordered = false) => Enumerable.Empty(); public IEnumerable GetAllValues(bool ordered = false) => Enumerable.Empty(); public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs index 4806e91ff0f..26f1628b720 100644 --- a/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs +++ b/src/Nethermind/Nethermind.Db/ReadOnlyDb.cs @@ -63,6 +63,8 @@ public KeyValuePair[] this[byte[][] keys] public IEnumerable> GetAll(bool ordered = false) => _memDb.GetAll(); + public IEnumerable GetAllKeys(bool ordered = false) => _memDb.GetAllKeys(); + public IEnumerable GetAllValues(bool ordered = false) => _memDb.GetAllValues(); public IWriteBatch StartWriteBatch() diff --git a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs index b917ac18cb4..74654f80b9e 100644 --- a/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs +++ b/src/Nethermind/Nethermind.Db/SimpleFilePublicKeyDb.cs @@ -99,6 +99,8 @@ public void Clear() public IEnumerable> GetAll(bool ordered = false) => _cache; + public IEnumerable GetAllKeys(bool ordered = false) => _cache.Keys; + public IEnumerable GetAllValues(bool ordered = false) => _cache.Values; public IWriteBatch StartWriteBatch() From da778b5ff18e345d3b0508ef0ca0bdd348795939 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 31 Oct 2023 14:21:30 +0100 Subject: [PATCH 03/41] adjust BlobTxStorage --- .../Nethermind.TxPool/BlobTxStorage.cs | 50 +++++++++++++++++++ .../Nethermind.TxPool/ITxStorage.cs | 3 ++ 2 files changed, 53 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 732221942e2..f28426a088a 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -18,6 +18,7 @@ public class BlobTxStorage : ITxStorage { private readonly IDb _fullBlobTxsDb; private readonly IDb _lightBlobTxsDb; + private readonly IDb _processedBlobTxsDb; private static readonly TxDecoder _txDecoder = new(); private static readonly LightTxDecoder _lightTxDecoder = new(); @@ -25,12 +26,14 @@ public BlobTxStorage() { _fullBlobTxsDb = new MemDb(); _lightBlobTxsDb = new MemDb(); + _processedBlobTxsDb = new MemDb(); } public BlobTxStorage(IColumnsDb database) { _fullBlobTxsDb = database.GetColumnDb(BlobTxsColumns.FullBlobTxs); _lightBlobTxsDb = database.GetColumnDb(BlobTxsColumns.LightBlobTxs); + _processedBlobTxsDb = database.GetColumnDb(BlobTxsColumns.ProcessedTxs); } public bool TryGet(in ValueHash256 hash, Address sender, in UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) @@ -112,6 +115,53 @@ private void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueHash256 hash, Sp timestamp.WriteBigEndian(txHashPrefixed); hash.Bytes.CopyTo(txHashPrefixed[32..]); } + + + public void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions) + { + if (blockBlobTransactions.Count == 0) + { + return; + } + + int contentLength = 0; + foreach (Transaction transaction in blockBlobTransactions) + { + contentLength += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + } + + RlpStream rlpStream = new(Rlp.LengthOfSequence(contentLength)); + rlpStream.StartSequence(contentLength); + foreach (Transaction transaction in blockBlobTransactions) + { + _txDecoder.Encode(rlpStream, transaction, RlpBehaviors.InMempoolForm); + } + + if (rlpStream.Data is not null) + { + _processedBlobTxsDb.Set(blockNumber, rlpStream.Data); + } + } + + public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions) + { + byte[]? bytes = _processedBlobTxsDb.Get(blockNumber); + + if (bytes is not null) + { + RlpStream rlpStream = new(bytes); + + blockBlobTransactions = _txDecoder.DecodeArray(rlpStream, RlpBehaviors.InMempoolForm); + return true; + + } + + blockBlobTransactions = default; + return false; + } + + public void DeleteBlobTransactionsFromBlock(long blockNumber) + => _processedBlobTxsDb.Delete(blockNumber); } internal static class UInt256Extensions diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 251d723b9e0..81a2a8b9f18 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -15,4 +15,7 @@ public interface ITxStorage IEnumerable GetAll(); void Add(Transaction transaction); void Delete(in ValueHash256 hash, in UInt256 timestamp); + bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions); + void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions); + void DeleteBlobTransactionsFromBlock(long blockNumber); } From 11c861a89fa9b04cda54f9ec987f18414c6d9445 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 31 Oct 2023 14:22:38 +0100 Subject: [PATCH 04/41] cosmetics --- src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs | 3 --- .../Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs index 6a5f838c54f..e890dba7445 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using FluentAssertions; using Nethermind.Blockchain.Receipts; -using Nethermind.Blockchain.Synchronization; using Nethermind.Consensus.Comparers; using Nethermind.Consensus.Processing; using Nethermind.Consensus.Rewards; @@ -14,13 +13,11 @@ using Nethermind.Core.Test.Builders; using Nethermind.Crypto; using Nethermind.Db; -using Nethermind.Db.Blooms; using Nethermind.Evm; using Nethermind.Evm.TransactionProcessing; using Nethermind.Logging; using Nethermind.Specs; using Nethermind.State; -using Nethermind.State.Repositories; using Nethermind.State.Witnesses; using Nethermind.Trie.Pruning; using Nethermind.TxPool; diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs index d8187852355..4b2e7ef3010 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs @@ -24,8 +24,6 @@ using Nethermind.JsonRpc.Modules.Parity; using Nethermind.Logging; using Nethermind.State; -using Nethermind.State.Repositories; -using Nethermind.Db.Blooms; using Nethermind.KeyStore; using Nethermind.Network; using Nethermind.Network.Contract.P2P; @@ -36,7 +34,6 @@ using Nethermind.TxPool; using NSubstitute; using NUnit.Framework; -using BlockTree = Nethermind.Blockchain.BlockTree; namespace Nethermind.JsonRpc.Test.Modules { From c2f456b927cc476aa8f34bc81eb44cba6aac4de4 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 31 Oct 2023 14:23:33 +0100 Subject: [PATCH 05/41] add logic for storing processed blob txs and removing finalized --- .../FinalizedTransactionsDbCleaner.cs | 69 +++++++++++++++++++ .../Nethermind.Merge.Plugin/MergePlugin.cs | 5 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 31 +++++++-- 3 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs diff --git a/src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs new file mode 100644 index 00000000000..c5cf730b659 --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Db; +using Nethermind.Logging; + +namespace Nethermind.Merge.Plugin; + +public class FinalizedTransactionsDbCleaner : IDisposable +{ + private readonly IManualBlockFinalizationManager _finalizationManager; + private readonly IDb _processedTxsDb; + private readonly ILogger _logger; + private long _lastFinalizedBlock = 0; + + public FinalizedTransactionsDbCleaner(IManualBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) + { + _finalizationManager = finalizationManager ?? throw new ArgumentNullException(nameof(finalizationManager)); + _processedTxsDb = processedTxsDb ?? throw new ArgumentNullException(nameof(processedTxsDb)); + _logger = logManager.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + + _finalizationManager.BlocksFinalized += OnBlocksFinalized; + } + + private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) + { + if (e.FinalizingBlock.Number > _lastFinalizedBlock) + { + Task.Run(() => CleanProcessedTransactionsDb(e.FinalizingBlock.Number)); + } + } + + private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) + { + try + { + List blocksInDb = new(); + foreach (byte[] key in _processedTxsDb.GetAllKeys()) + { + blocksInDb.Add(key.ToLongFromBigEndianByteArrayWithoutLeadingZeros()); + } + + foreach (long blockNumber in blocksInDb) + { + if (newlyFinalizedBlockNumber >= blockNumber) + { + _processedTxsDb.Delete(blockNumber); + } + } + + _lastFinalizedBlock = newlyFinalizedBlockNumber; + } + catch (Exception exception) + { + if (_logger.IsError) _logger.Error($"Couldn't correctly clean db with processed transactions. Newly finalized block {newlyFinalizedBlockNumber}, last finalized block: {_lastFinalizedBlock}", exception); + } + } + + public void Dispose() + { + _finalizationManager.BlocksFinalized -= OnBlocksFinalized; + } +} diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 65617408097..941ddfadcc3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading; @@ -16,7 +15,6 @@ using Nethermind.Consensus.Rewards; using Nethermind.Consensus.Validators; using Nethermind.Core; -using Nethermind.Core.Crypto; using Nethermind.Core.Exceptions; using Nethermind.Db; using Nethermind.Facade.Proxy; @@ -26,7 +24,6 @@ using Nethermind.Logging; using Nethermind.Merge.Plugin.BlockProduction; using Nethermind.Merge.Plugin.BlockProduction.Boost; -using Nethermind.Merge.Plugin.Data; using Nethermind.Merge.Plugin.GC; using Nethermind.Merge.Plugin.Handlers; using Nethermind.Merge.Plugin.InvalidChainTracker; @@ -101,6 +98,8 @@ public virtual Task Init(INethermindApi nethermindApi) _api.PoSSwitcher = _poSSwitcher; _api.DisposeStack.Push(_invalidChainTracker); _blockFinalizationManager = new ManualBlockFinalizationManager(); + FinalizedTransactionsDbCleaner finalizedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); + _api.DisposeStack.Push(finalizedTransactionsDbCleaner); _api.RewardCalculatorSource = new MergeRewardCalculatorSource( _api.RewardCalculatorSource ?? NoBlockRewards.Instance, _poSSwitcher); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 4ee0cf0d5bf..a2582c1bac9 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -43,6 +43,7 @@ public class TxPool : ITxPool, IDisposable private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; + private readonly ITxStorage _blobTxStorage; private readonly IChainHeadInfoProvider _headInfo; private readonly ITxPoolConfig _txPoolConfig; @@ -87,6 +88,7 @@ public TxPool(IEthereumEcdsa ecdsa, bool thereIsPriorityContract = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; _accounts = _headInfo.AccountStateProvider; @@ -198,7 +200,7 @@ private void ProcessNewHeads() try { ReAddReorganisedTransactions(args.PreviousBlock); - RemoveProcessedTransactions(args.Block.Transactions); + RemoveProcessedTransactions(args.Block); UpdateBuckets(); _broadcaster.BroadcastPersistentTxs(); Metrics.TransactionCount = _transactions.Count; @@ -235,15 +237,27 @@ private void ReAddReorganisedTransactions(Block? previousBlock) _hashCache.Delete(tx.Hash!); SubmitTx(tx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); } + + if (_blobTxStorage.TryGetBlobTransactionsFromBlock(previousBlock.Number, out Transaction[]? blobTxs) && blobTxs is not null) + { + foreach (Transaction blobTx in blobTxs) + { + _hashCache.Delete(blobTx.Hash!); + SubmitTx(blobTx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); + } + + _blobTxStorage.DeleteBlobTransactionsFromBlock(previousBlock.Number); + } } } - private void RemoveProcessedTransactions(Transaction[] blockTransactions) + private void RemoveProcessedTransactions(Block block) { + Transaction[] blockTransactions = block.Transactions; + List? blobTxs = null; long discoveredForPendingTxs = 0; long discoveredForHashCache = 0; long eip1559Txs = 0; - long blobTxs = 0; long blobs = 0; for (int i = 0; i < blockTransactions.Length; i++) @@ -268,18 +282,25 @@ private void RemoveProcessedTransactions(Transaction[] blockTransactions) if (transaction.SupportsBlobs) { - blobTxs++; + blobTxs ??= new List(Eip4844Constants.MaxBlobsPerBlock); + blobTxs.Add(transaction); + blobs += transaction.BlobVersionedHashes?.Length ?? 0; } } + if (blobTxs is not null) + { + _blobTxStorage.AddBlobTransactionsFromBlock(block.Number, blobTxs); + } + long transactionsInBlock = blockTransactions.Length; if (transactionsInBlock != 0) { Metrics.DarkPoolRatioLevel1 = (float)discoveredForHashCache / transactionsInBlock; Metrics.DarkPoolRatioLevel2 = (float)discoveredForPendingTxs / transactionsInBlock; Metrics.Eip1559TransactionsRatio = (float)eip1559Txs / transactionsInBlock; - Metrics.BlobTransactionsInBlock = blobTxs; + Metrics.BlobTransactionsInBlock = blobTxs?.Count ?? 0; Metrics.BlobsInBlock = blobs; } } From ea0ea11b779cfbc2b9a973632a46bd39d6678a9a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 20 Jul 2023 23:38:57 +0200 Subject: [PATCH 06/41] rename --- src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs | 4 ++-- ...ionsDbCleaner.cs => ProcessedTransactionsDbCleaner.cs} | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) rename src/Nethermind/Nethermind.Merge.Plugin/{FinalizedTransactionsDbCleaner.cs => ProcessedTransactionsDbCleaner.cs} (90%) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 941ddfadcc3..65553e42f09 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -98,8 +98,8 @@ public virtual Task Init(INethermindApi nethermindApi) _api.PoSSwitcher = _poSSwitcher; _api.DisposeStack.Push(_invalidChainTracker); _blockFinalizationManager = new ManualBlockFinalizationManager(); - FinalizedTransactionsDbCleaner finalizedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); - _api.DisposeStack.Push(finalizedTransactionsDbCleaner); + ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); + _api.DisposeStack.Push(processedTransactionsDbCleaner); _api.RewardCalculatorSource = new MergeRewardCalculatorSource( _api.RewardCalculatorSource ?? NoBlockRewards.Instance, _poSSwitcher); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs similarity index 90% rename from src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs rename to src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index c5cf730b659..6e56b54817b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/FinalizedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -12,14 +12,14 @@ namespace Nethermind.Merge.Plugin; -public class FinalizedTransactionsDbCleaner : IDisposable +public class ProcessedTransactionsDbCleaner : IDisposable { private readonly IManualBlockFinalizationManager _finalizationManager; private readonly IDb _processedTxsDb; private readonly ILogger _logger; private long _lastFinalizedBlock = 0; - public FinalizedTransactionsDbCleaner(IManualBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) + public ProcessedTransactionsDbCleaner(IManualBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) { _finalizationManager = finalizationManager ?? throw new ArgumentNullException(nameof(finalizationManager)); _processedTxsDb = processedTxsDb ?? throw new ArgumentNullException(nameof(processedTxsDb)); @@ -30,9 +30,9 @@ public FinalizedTransactionsDbCleaner(IManualBlockFinalizationManager finalizati private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) { - if (e.FinalizingBlock.Number > _lastFinalizedBlock) + if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > _lastFinalizedBlock) { - Task.Run(() => CleanProcessedTransactionsDb(e.FinalizingBlock.Number)); + Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); } } From aa5f93c5fc2e92a58a703d9e74679d9c76c8376b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 10:45:32 +0100 Subject: [PATCH 07/41] fix --- src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index c4b8c933b0b..188aaa09eab 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -25,4 +25,15 @@ public bool TryGet(in ValueHash256 hash, Address sender, in UInt256 timestamp, [ public void Add(Transaction transaction) { } public void Delete(in ValueHash256 hash, in UInt256 timestamp) { } + + public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions) + { + blockBlobTransactions = default; + return false; + } + + public void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions) { } + + public void DeleteBlobTransactionsFromBlock(long blockNumber) + { } } From 6ac8ecbbb5988907b784f4fb435b7ec4f2b0c8c7 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 10:54:46 +0100 Subject: [PATCH 08/41] add tests --- .../ProcessedTransactionsDbCleanerTests.cs | 60 +++++++++++++++++++ .../TxPoolTests.Blobs.cs | 49 +++++++++++++++ .../Nethermind.TxPool.Test/TxPoolTests.cs | 4 +- 3 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs new file mode 100644 index 00000000000..7a1e7b816ab --- /dev/null +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Db; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Specs; +using Nethermind.TxPool; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.Merge.Plugin.Test; + +[TestFixture] +public class ProcessedTransactionsDbCleanerTests +{ + private readonly ILogManager _logManager = LimboLogs.Instance; + private readonly ISpecProvider _specProvider = MainnetSpecProvider.Instance; + + [Test] + public void should_remove_processed_txs_from_db_after_finalization([Values(0, 1, 42, 358)] long blockOfTxs, [Values(1, 42, 358)] long finalizedBlock) + { + Transaction GetTx(PrivateKey sender) + { + return Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(new EthereumEcdsa(_specProvider.ChainId, _logManager), sender).TestObject; + } + + IColumnsDb columnsDb = new MemColumnsDb(BlobTxsColumns.ProcessedTxs); + BlobTxStorage blobTxStorage = new(columnsDb); + Transaction[] txs = { GetTx(TestItem.PrivateKeyA), GetTx(TestItem.PrivateKeyB) }; + + blobTxStorage.AddBlobTransactionsFromBlock(blockOfTxs, txs); + + blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out Transaction[]? returnedTxs).Should().BeTrue(); + returnedTxs!.Length.Should().Be(2); + + IManualBlockFinalizationManager finalizationManager = Substitute.For(); + ProcessedTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _logManager); + + finalizationManager.BlocksFinalized += Raise.EventWith( + new FinalizeEventArgs(Build.A.BlockHeader.TestObject, + Build.A.BlockHeader.WithNumber(finalizedBlock).TestObject)); + + Thread.Sleep(100); + + blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out returnedTxs).Should().Be(blockOfTxs > finalizedBlock); + } +} diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index beea6b95e6c..530ca99996a 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -6,6 +6,7 @@ using Nethermind.Core; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; using Nethermind.Evm; using Nethermind.Int256; using NUnit.Framework; @@ -490,5 +491,53 @@ public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int ex .TestObject; blobTx.GetLength().Should().Be(expectedLength); } + + [Test] + public async Task should_add_processed_txs_to_db() + { + Transaction GetTx(PrivateKey sender) + { + return Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, sender).TestObject; + } + + const long blockNumber = 358; + + BlobTxStorage blobTxStorage = new(); + _txPool = CreatePool(new TxPoolConfig(){ Size = 128, BlobSupportEnabled = true }, GetCancunSpecProvider(), txStorage: blobTxStorage); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); + + Transaction[] txs = { GetTx(TestItem.PrivateKeyA), GetTx(TestItem.PrivateKeyB) }; + + _txPool.SubmitTx(txs[0], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(txs[1], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(txs.Length); + _stateProvider.IncrementNonce(TestItem.AddressA); + _stateProvider.IncrementNonce(TestItem.AddressB); + + Block block = Build.A.Block.WithNumber(blockNumber).WithTransactions(txs).TestObject; + + await RaiseBlockAddedToMainAndWaitForTransactions(txs.Length, block); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(0); + + blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out Transaction[] returnedTxs).Should().BeTrue(); + returnedTxs.Length.Should().Be(txs.Length); + returnedTxs.Should().BeEquivalentTo(txs, options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex + + blobTxStorage.DeleteBlobTransactionsFromBlock(blockNumber); + blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out returnedTxs).Should().BeFalse(); + } } } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 6ad0f609cc9..4c8d9c15c62 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1803,11 +1803,11 @@ private Transaction GetTransaction(UInt256 nonce, long gasLimit, UInt256 gasPric .SignedAndResolved(_ethereumEcdsa, privateKey) .TestObject; - private async Task RaiseBlockAddedToMainAndWaitForTransactions(int txCount) + private async Task RaiseBlockAddedToMainAndWaitForTransactions(int txCount, Block block = null) { SemaphoreSlim semaphoreSlim = new(0, txCount); _txPool.NewPending += (o, e) => semaphoreSlim.Release(); - _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(Build.A.Block.TestObject)); + _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject)); for (int i = 0; i < txCount; i++) { await semaphoreSlim.WaitAsync(10); From ef83a9c03e57309f97d45b9f084d9856295c0850 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 11:30:42 +0100 Subject: [PATCH 09/41] cosmetics --- .../Nethermind.TxPool/BlobTxStorage.cs | 69 +++++++++---------- 1 file changed, 34 insertions(+), 35 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index f28426a088a..d8ba8ec081d 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -16,11 +16,11 @@ namespace Nethermind.TxPool; public class BlobTxStorage : ITxStorage { + private static readonly TxDecoder _txDecoder = new(); + private static readonly LightTxDecoder _lightTxDecoder = new(); private readonly IDb _fullBlobTxsDb; private readonly IDb _lightBlobTxsDb; private readonly IDb _processedBlobTxsDb; - private static readonly TxDecoder _txDecoder = new(); - private static readonly LightTxDecoder _lightTxDecoder = new(); public BlobTxStorage() { @@ -84,39 +84,6 @@ public void Delete(in ValueHash256 hash, in UInt256 timestamp) _lightBlobTxsDb.Remove(hash.BytesAsSpan); } - private static bool TryDecodeFullTx(byte[]? txBytes, Address sender, out Transaction? transaction) - { - if (txBytes is not null) - { - RlpStream rlpStream = new(txBytes); - transaction = Rlp.Decode(rlpStream, RlpBehaviors.InMempoolForm); - transaction.SenderAddress = sender; - return true; - } - - transaction = default; - return false; - } - - private static bool TryDecodeLightTx(byte[]? txBytes, out LightTransaction? lightTx) - { - if (txBytes is not null) - { - lightTx = _lightTxDecoder.Decode(txBytes); - return true; - } - - lightTx = default; - return false; - } - - private void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueHash256 hash, Span txHashPrefixed) - { - timestamp.WriteBigEndian(txHashPrefixed); - hash.Bytes.CopyTo(txHashPrefixed[32..]); - } - - public void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions) { if (blockBlobTransactions.Count == 0) @@ -162,6 +129,38 @@ public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? public void DeleteBlobTransactionsFromBlock(long blockNumber) => _processedBlobTxsDb.Delete(blockNumber); + + private static bool TryDecodeFullTx(byte[]? txBytes, Address sender, out Transaction? transaction) + { + if (txBytes is not null) + { + RlpStream rlpStream = new(txBytes); + transaction = Rlp.Decode(rlpStream, RlpBehaviors.InMempoolForm); + transaction.SenderAddress = sender; + return true; + } + + transaction = default; + return false; + } + + private static bool TryDecodeLightTx(byte[]? txBytes, out LightTransaction? lightTx) + { + if (txBytes is not null) + { + lightTx = _lightTxDecoder.Decode(txBytes); + return true; + } + + lightTx = default; + return false; + } + + private void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueHash256 hash, Span txHashPrefixed) + { + timestamp.WriteBigEndian(txHashPrefixed); + hash.Bytes.CopyTo(txHashPrefixed[32..]); + } } internal static class UInt256Extensions From 30bffaa1a4078caa685e60ca8a5171391fe4e3c6 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 11:37:41 +0100 Subject: [PATCH 10/41] add sender recovery for reorganized blob txs --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 5 +++-- src/Nethermind/Nethermind.TxPool/TxPool.cs | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 530ca99996a..e899ea43610 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -533,8 +533,9 @@ Transaction GetTx(PrivateKey sender) blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out Transaction[] returnedTxs).Should().BeTrue(); returnedTxs.Length.Should().Be(txs.Length); returnedTxs.Should().BeEquivalentTo(txs, options => options - .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... - .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex) // ...as well as PoolIndex + .Excluding(t => t.SenderAddress)); // sender is recovered later, it is not returned from db blobTxStorage.DeleteBlobTransactionsFromBlock(blockNumber); blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out returnedTxs).Should().BeFalse(); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index a2582c1bac9..ece66032f9a 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -43,6 +43,7 @@ public class TxPool : ITxPool, IDisposable private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; + private readonly IEthereumEcdsa _ecdsa; private readonly ITxStorage _blobTxStorage; private readonly IChainHeadInfoProvider _headInfo; private readonly ITxPoolConfig _txPoolConfig; @@ -88,6 +89,7 @@ public TxPool(IEthereumEcdsa ecdsa, bool thereIsPriorityContract = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _ecdsa = ecdsa ?? throw new ArgumentNullException(nameof(ecdsa)); _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; @@ -243,6 +245,7 @@ private void ReAddReorganisedTransactions(Block? previousBlock) foreach (Transaction blobTx in blobTxs) { _hashCache.Delete(blobTx.Hash!); + blobTx.SenderAddress ??= _ecdsa.RecoverAddress(blobTx); SubmitTx(blobTx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); } From eea689df9d5c9e63cc3cfa23e4311c5385582b58 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 11:43:48 +0100 Subject: [PATCH 11/41] requested change --- .../Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index 6e56b54817b..4bf8734059c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -14,12 +14,12 @@ namespace Nethermind.Merge.Plugin; public class ProcessedTransactionsDbCleaner : IDisposable { - private readonly IManualBlockFinalizationManager _finalizationManager; + private readonly IBlockFinalizationManager _finalizationManager; private readonly IDb _processedTxsDb; private readonly ILogger _logger; private long _lastFinalizedBlock = 0; - public ProcessedTransactionsDbCleaner(IManualBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) + public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) { _finalizationManager = finalizationManager ?? throw new ArgumentNullException(nameof(finalizationManager)); _processedTxsDb = processedTxsDb ?? throw new ArgumentNullException(nameof(processedTxsDb)); From 42fcd1b90358f6e81fc99e282410b9a46377dc32 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 11:49:58 +0100 Subject: [PATCH 12/41] one more place --- .../ProcessedTransactionsDbCleanerTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs index 7a1e7b816ab..fbcc579a7bd 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -46,7 +46,7 @@ Transaction GetTx(PrivateKey sender) blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out Transaction[]? returnedTxs).Should().BeTrue(); returnedTxs!.Length.Should().Be(2); - IManualBlockFinalizationManager finalizationManager = Substitute.For(); + IBlockFinalizationManager finalizationManager = Substitute.For(); ProcessedTransactionsDbCleaner dbCleaner = new(finalizationManager, columnsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _logManager); finalizationManager.BlocksFinalized += Raise.EventWith( From 201f590414faf7489643c4dd6acc9ce0da62702f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 2 Nov 2023 11:51:26 +0100 Subject: [PATCH 13/41] postmerge fix --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index ece66032f9a..7a5d7b2f0fa 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -285,7 +285,7 @@ private void RemoveProcessedTransactions(Block block) if (transaction.SupportsBlobs) { - blobTxs ??= new List(Eip4844Constants.MaxBlobsPerBlock); + blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); blobTxs.Add(transaction); blobs += transaction.BlobVersionedHashes?.Length ?? 0; From 7c2c556aa602c5faddc5ae407687f65df088217d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 3 Nov 2023 16:22:06 +0100 Subject: [PATCH 14/41] fix saving of processed blob txs --- .../Nethermind.TxPool/NullBlobTxStorage.cs | 3 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 29 ++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index 188aaa09eab..2a0b8bda575 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -34,6 +34,5 @@ public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? public void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions) { } - public void DeleteBlobTransactionsFromBlock(long blockNumber) - { } + public void DeleteBlobTransactionsFromBlock(long blockNumber) { } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 7a5d7b2f0fa..8b1a797be57 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -265,30 +265,33 @@ private void RemoveProcessedTransactions(Block block) for (int i = 0; i < blockTransactions.Length; i++) { - Transaction transaction = blockTransactions[i]; - Hash256 txHash = transaction.Hash ?? throw new ArgumentException("Hash was unexpectedly null!"); + Transaction blockTx = blockTransactions[i]; + Hash256 txHash = blockTx.Hash ?? throw new ArgumentException("Hash was unexpectedly null!"); - if (!IsKnown(txHash)) + if (blockTx.Supports1559) { - discoveredForHashCache++; + eip1559Txs++; } - if (!RemoveIncludedTransaction(transaction)) + if (blockTx.SupportsBlobs) { - discoveredForPendingTxs++; + blobs += blockTx.BlobVersionedHashes?.Length ?? 0; + + if (_blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) + { + blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); + blobTxs.Add(fullBlobTx); + } } - if (transaction.Supports1559) + if (!IsKnown(txHash)) { - eip1559Txs++; + discoveredForHashCache++; } - if (transaction.SupportsBlobs) + if (!RemoveIncludedTransaction(blockTx)) { - blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); - blobTxs.Add(transaction); - - blobs += transaction.BlobVersionedHashes?.Length ?? 0; + discoveredForPendingTxs++; } } From 48b93c32c80ec6b3078f5c8a5de833555fcbdcae Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 6 Nov 2023 19:24:02 +0100 Subject: [PATCH 15/41] optimize saving processed txs --- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index d8ba8ec081d..0c386376ed4 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -8,6 +8,7 @@ using DotNetty.Buffers; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Db; using Nethermind.Int256; using Nethermind.Serialization.Rlp; @@ -97,17 +98,15 @@ public void AddBlobTransactionsFromBlock(long blockNumber, IList bl contentLength += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); } - RlpStream rlpStream = new(Rlp.LengthOfSequence(contentLength)); + IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(Rlp.LengthOfSequence(contentLength)); + using NettyRlpStream rlpStream = new(byteBuffer); rlpStream.StartSequence(contentLength); foreach (Transaction transaction in blockBlobTransactions) { _txDecoder.Encode(rlpStream, transaction, RlpBehaviors.InMempoolForm); } - if (rlpStream.Data is not null) - { - _processedBlobTxsDb.Set(blockNumber, rlpStream.Data); - } + _processedBlobTxsDb.Set(blockNumber, byteBuffer.Array); } public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions) From ebe2ec597e1f432528ac7ef70bae9807cc3a428a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 7 Nov 2023 13:19:00 +0100 Subject: [PATCH 16/41] readd blob support config --- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 3 +++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 9 +++++++-- src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 9f7d9ef9d0b..388d6d9c75f 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -28,6 +28,9 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "512", Description = "Max number of full blob transactions stored in memory. Used only if persistent storage is disabled")] int InMemoryBlobPoolSize { get; set; } + [ConfigItem(DefaultValue = "true", Description = "If true, all processed blob transactions would be stored in persistent db until finalization to readd them to the pool in case of chain reorganization")] + bool BlobReorgsSupportEnabled { get; set; } + [ConfigItem(DefaultValue = "0", Description = "Max number of pending transactions per single sender. Set it to 0 to disable the limit.")] int MaxPendingTxsPerSender { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 8b1a797be57..6747b9886f5 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -47,6 +47,7 @@ public class TxPool : ITxPool, IDisposable private readonly ITxStorage _blobTxStorage; private readonly IChainHeadInfoProvider _headInfo; private readonly ITxPoolConfig _txPoolConfig; + private readonly bool _blobReorgsSupportEnabled; private readonly ILogger _logger; @@ -93,6 +94,7 @@ public TxPool(IEthereumEcdsa ecdsa, _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; + _blobReorgsSupportEnabled = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true, BlobReorgsSupportEnabled: true }; _accounts = _headInfo.AccountStateProvider; _specProvider = _headInfo.SpecProvider; @@ -240,7 +242,9 @@ private void ReAddReorganisedTransactions(Block? previousBlock) SubmitTx(tx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); } - if (_blobTxStorage.TryGetBlobTransactionsFromBlock(previousBlock.Number, out Transaction[]? blobTxs) && blobTxs is not null) + if (_blobReorgsSupportEnabled + && _blobTxStorage.TryGetBlobTransactionsFromBlock(previousBlock.Number, out Transaction[]? blobTxs) + && blobTxs is not null) { foreach (Transaction blobTx in blobTxs) { @@ -277,7 +281,8 @@ private void RemoveProcessedTransactions(Block block) { blobs += blockTx.BlobVersionedHashes?.Length ?? 0; - if (_blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) + if (_blobReorgsSupportEnabled + && _blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) { blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); blobTxs.Add(fullBlobTx); diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 071ed94dd04..34f2fe81caf 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -14,6 +14,7 @@ public class TxPoolConfig : ITxPoolConfig // every day about 21600 blobs will be included (7200 blocks per day * 3 blob target) public int BlobCacheSize { get; set; } = 256; public int InMemoryBlobPoolSize { get; set; } = 512; // it is used when persistent pool is disabled + public bool BlobReorgsSupportEnabled { get; set; } = true; public int MaxPendingTxsPerSender { get; set; } = 0; public int MaxPendingBlobTxsPerSender { get; set; } = 16; public int HashCacheSize { get; set; } = 512 * 1024; From d18ea5d0472809e3cdea7a433c50f49bb12a8ae0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 7 Nov 2023 13:45:22 +0100 Subject: [PATCH 17/41] fix whitespace --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index e899ea43610..c3bdf9e0377 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -508,7 +508,7 @@ Transaction GetTx(PrivateKey sender) const long blockNumber = 358; BlobTxStorage blobTxStorage = new(); - _txPool = CreatePool(new TxPoolConfig(){ Size = 128, BlobSupportEnabled = true }, GetCancunSpecProvider(), txStorage: blobTxStorage); + _txPool = CreatePool(new TxPoolConfig() { Size = 128, BlobSupportEnabled = true }, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); From 33ad928b0c9e85dfc601b57909d03af44458dfa7 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 7 Nov 2023 14:08:20 +0100 Subject: [PATCH 18/41] fix test --- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index c3bdf9e0377..681be70e92d 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -508,7 +508,14 @@ Transaction GetTx(PrivateKey sender) const long blockNumber = 358; BlobTxStorage blobTxStorage = new(); - _txPool = CreatePool(new TxPoolConfig() { Size = 128, BlobSupportEnabled = true }, GetCancunSpecProvider(), txStorage: blobTxStorage); + ITxPoolConfig txPoolConfig = new TxPoolConfig() + { + Size = 128, + BlobSupportEnabled = true, + PersistentBlobStorageEnabled = true, + BlobReorgsSupportEnabled = true + }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); From 3ef6efce2dbf8ca78414e3bd8ddf19cde6a4d73e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 7 Nov 2023 14:46:04 +0100 Subject: [PATCH 19/41] fix tests parallelism? --- .../ProcessedTransactionsDbCleanerTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs index fbcc579a7bd..09612a29235 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -19,6 +19,7 @@ namespace Nethermind.Merge.Plugin.Test; [TestFixture] +[Parallelizable(ParallelScope.All)] public class ProcessedTransactionsDbCleanerTests { private readonly ILogManager _logManager = LimboLogs.Instance; From 30a5a24f9bc889411a6d32a035baa415713acebd Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 9 Nov 2023 13:43:13 +0100 Subject: [PATCH 20/41] add more logs --- .../ProcessedTransactionsDbCleaner.cs | 3 +++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 14 ++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index 4bf8734059c..c78c8f5cb45 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -46,13 +46,16 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) blocksInDb.Add(key.ToLongFromBigEndianByteArrayWithoutLeadingZeros()); } + if (_logger.IsTrace) _logger.Trace($"There are processed blob txs from {blocksInDb.Count} blocks in ProcessedTxs db"); foreach (long blockNumber in blocksInDb) { if (newlyFinalizedBlockNumber >= blockNumber) { + if (_logger.IsTrace) _logger.Trace($"Cleaning processed blob txs from block {blockNumber}"); _processedTxsDb.Delete(blockNumber); } } + if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); _lastFinalizedBlock = newlyFinalizedBlockNumber; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 6747b9886f5..ee832696db9 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -248,10 +248,12 @@ private void ReAddReorganisedTransactions(Block? previousBlock) { foreach (Transaction blobTx in blobTxs) { + if (_logger.IsTrace) _logger.Trace($"Readded tx {blobTx.Hash} from reorged block {previousBlock.Number} (hash {previousBlock.Hash}) to blob pool"); _hashCache.Delete(blobTx.Hash!); blobTx.SenderAddress ??= _ecdsa.RecoverAddress(blobTx); SubmitTx(blobTx, isEip155Enabled ? TxHandlingOptions.None : TxHandlingOptions.PreEip155Signing); } + if (_logger.IsDebug) _logger.Debug($"Readded txs from reorged block {previousBlock.Number} (hash {previousBlock.Hash}) to blob pool"); _blobTxStorage.DeleteBlobTransactionsFromBlock(previousBlock.Number); } @@ -281,11 +283,15 @@ private void RemoveProcessedTransactions(Block block) { blobs += blockTx.BlobVersionedHashes?.Length ?? 0; - if (_blobReorgsSupportEnabled - && _blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) + if (_blobReorgsSupportEnabled) { - blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); - blobTxs.Add(fullBlobTx); + if (_blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) + { + if (_logger.IsTrace) _logger.Trace($"Saved processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db"); + blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); + blobTxs.Add(fullBlobTx); + } + else if (_logger.IsTrace) _logger.Trace($"Skipped adding processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db - not found in blob pool"); } } From a1e7ae46be52a356ec21abb8397ccc4c0472b62a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 8 Nov 2023 11:55:21 +0100 Subject: [PATCH 21/41] fix blob txs metric --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index ee832696db9..50da172de01 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -263,10 +263,11 @@ private void ReAddReorganisedTransactions(Block? previousBlock) private void RemoveProcessedTransactions(Block block) { Transaction[] blockTransactions = block.Transactions; - List? blobTxs = null; + List? blobTxsToSave = null; long discoveredForPendingTxs = 0; long discoveredForHashCache = 0; long eip1559Txs = 0; + long blobTxs = 0; long blobs = 0; for (int i = 0; i < blockTransactions.Length; i++) @@ -281,6 +282,7 @@ private void RemoveProcessedTransactions(Block block) if (blockTx.SupportsBlobs) { + blobTxs++; blobs += blockTx.BlobVersionedHashes?.Length ?? 0; if (_blobReorgsSupportEnabled) @@ -288,8 +290,8 @@ private void RemoveProcessedTransactions(Block block) if (_blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) { if (_logger.IsTrace) _logger.Trace($"Saved processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db"); - blobTxs ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); - blobTxs.Add(fullBlobTx); + blobTxsToSave ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); + blobTxsToSave.Add(fullBlobTx); } else if (_logger.IsTrace) _logger.Trace($"Skipped adding processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db - not found in blob pool"); } @@ -306,9 +308,9 @@ private void RemoveProcessedTransactions(Block block) } } - if (blobTxs is not null) + if (blobTxsToSave is not null) { - _blobTxStorage.AddBlobTransactionsFromBlock(block.Number, blobTxs); + _blobTxStorage.AddBlobTransactionsFromBlock(block.Number, blobTxsToSave); } long transactionsInBlock = blockTransactions.Length; @@ -317,7 +319,7 @@ private void RemoveProcessedTransactions(Block block) Metrics.DarkPoolRatioLevel1 = (float)discoveredForHashCache / transactionsInBlock; Metrics.DarkPoolRatioLevel2 = (float)discoveredForPendingTxs / transactionsInBlock; Metrics.Eip1559TransactionsRatio = (float)eip1559Txs / transactionsInBlock; - Metrics.BlobTransactionsInBlock = blobTxs?.Count ?? 0; + Metrics.BlobTransactionsInBlock = blobTxs; Metrics.BlobsInBlock = blobs; } } From 31922a8828300a3305fc97c20796cc633313a8bb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 8 Nov 2023 11:55:21 +0100 Subject: [PATCH 22/41] refactor BlobTxStorage --- .../Nethermind.TxPool/BlobTxStorage.cs | 62 ++++++++++++------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 0c386376ed4..17077fc622b 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -8,7 +8,6 @@ using DotNetty.Buffers; using Nethermind.Core; using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; using Nethermind.Db; using Nethermind.Int256; using Nethermind.Serialization.Rlp; @@ -67,12 +66,7 @@ public void Add(Transaction transaction) Span txHashPrefixed = stackalloc byte[64]; GetHashPrefixedByTimestamp(transaction.Timestamp, transaction.Hash, txHashPrefixed); - int length = _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); - IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(length); - using NettyRlpStream rlpStream = new(byteBuffer); - rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); - - _fullBlobTxsDb.PutSpan(txHashPrefixed, byteBuffer.AsSpan()); + _fullBlobTxsDb.PutSpan(txHashPrefixed, EncodeTx(transaction)); _lightBlobTxsDb.Set(transaction.Hash, _lightTxDecoder.Encode(transaction)); } @@ -92,21 +86,7 @@ public void AddBlobTransactionsFromBlock(long blockNumber, IList bl return; } - int contentLength = 0; - foreach (Transaction transaction in blockBlobTransactions) - { - contentLength += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); - } - - IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(Rlp.LengthOfSequence(contentLength)); - using NettyRlpStream rlpStream = new(byteBuffer); - rlpStream.StartSequence(contentLength); - foreach (Transaction transaction in blockBlobTransactions) - { - _txDecoder.Encode(rlpStream, transaction, RlpBehaviors.InMempoolForm); - } - - _processedBlobTxsDb.Set(blockNumber, byteBuffer.Array); + _processedBlobTxsDb.Set(blockNumber, EncodeTxs(blockBlobTransactions)); } public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions) @@ -116,10 +96,8 @@ public bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? if (bytes is not null) { RlpStream rlpStream = new(bytes); - blockBlobTransactions = _txDecoder.DecodeArray(rlpStream, RlpBehaviors.InMempoolForm); return true; - } blockBlobTransactions = default; @@ -160,6 +138,42 @@ private void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueHash256 hash, Sp timestamp.WriteBigEndian(txHashPrefixed); hash.Bytes.CopyTo(txHashPrefixed[32..]); } + + private Span EncodeTx(Transaction transaction) + { + int length = _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(length); + using NettyRlpStream rlpStream = new(byteBuffer); + rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); + + return byteBuffer.AsSpan(); + } + + private byte[] EncodeTxs(IList blockBlobTransactions) + { + int contentLength = GetLength(blockBlobTransactions); + + IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(Rlp.LengthOfSequence(contentLength)); + using NettyRlpStream rlpStream = new(byteBuffer); + rlpStream.StartSequence(contentLength); + foreach (Transaction transaction in blockBlobTransactions) + { + _txDecoder.Encode(rlpStream, transaction, RlpBehaviors.InMempoolForm); + } + + return byteBuffer.Array; + } + + private int GetLength(IList blockBlobTransactions) + { + int contentLength = 0; + foreach (Transaction transaction in blockBlobTransactions) + { + contentLength += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + } + + return contentLength; + } } internal static class UInt256Extensions From 1849136b6c98fb7371d7a8f964f1cc4558c14d92 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 7 Dec 2023 10:00:16 +0100 Subject: [PATCH 23/41] optimize collecting blob txs from processed block --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 2f0ffb86905..c1483e85598 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -264,7 +264,7 @@ private void ReAddReorganisedTransactions(Block? previousBlock) private void RemoveProcessedTransactions(Block block) { Transaction[] blockTransactions = block.Transactions; - List? blobTxsToSave = null; + using ArrayPoolList blobTxsToSave = new(Eip4844Constants.GetMaxBlobsPerBlock()); long discoveredForPendingTxs = 0; long discoveredForHashCache = 0; long eip1559Txs = 0; @@ -291,7 +291,6 @@ private void RemoveProcessedTransactions(Block block) if (_blobTransactions.TryGetValue(blockTx.Hash, out Transaction? fullBlobTx)) { if (_logger.IsTrace) _logger.Trace($"Saved processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db"); - blobTxsToSave ??= new List(Eip4844Constants.GetMaxBlobsPerBlock()); blobTxsToSave.Add(fullBlobTx); } else if (_logger.IsTrace) _logger.Trace($"Skipped adding processed blob tx {blockTx.Hash} from block {block.Number} to ProcessedTxs db - not found in blob pool"); @@ -309,7 +308,7 @@ private void RemoveProcessedTransactions(Block block) } } - if (blobTxsToSave is not null) + if (blobTxsToSave.Count > 0) { _blobTxStorage.AddBlobTransactionsFromBlock(block.Number, blobTxsToSave); } From 2b733123ec5026b7e623c45fdf8556104db263b9 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 7 Dec 2023 13:15:51 +0100 Subject: [PATCH 24/41] Create IBlobTxStorage and separate blob specific methods from ITxStorage --- src/Nethermind/Nethermind.Api/IApiWithStores.cs | 2 +- src/Nethermind/Nethermind.Api/NethermindApi.cs | 2 +- .../Nethermind.TxPool.Test/TxPoolTests.cs | 2 +- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 2 +- src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs | 14 ++++++++++++++ src/Nethermind/Nethermind.TxPool/ITxStorage.cs | 3 --- .../Nethermind.TxPool/NullBlobTxStorage.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 ++-- 8 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 12fe585bc59..8a5bfa981d5 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -15,7 +15,7 @@ namespace Nethermind.Api { public interface IApiWithStores : IBasicApi { - ITxStorage? BlobTxStorage { get; set; } + IBlobTxStorage? BlobTxStorage { get; set; } IBlockTree? BlockTree { get; set; } IBloomStorage? BloomStorage { get; set; } IChainLevelInfoRepository? ChainLevelInfoRepository { get; set; } diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index d83305c6cb8..616eaba6172 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -104,7 +104,7 @@ public IBlockchainBridge CreateBlockchainBridge() } public IAbiEncoder AbiEncoder { get; } = Nethermind.Abi.AbiEncoder.Instance; - public ITxStorage? BlobTxStorage { get; set; } + public IBlobTxStorage? BlobTxStorage { get; set; } public IBlockchainProcessor? BlockchainProcessor { get; set; } public CompositeBlockPreprocessorStep BlockPreprocessor { get; } = new(); public IBlockProcessingQueue? BlockProcessingQueue { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 4c8d9c15c62..e373cb57363 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1657,7 +1657,7 @@ private TxPool CreatePool( ISpecProvider specProvider = null, ChainHeadInfoProvider chainHeadInfoProvider = null, IIncomingTxFilter incomingTxFilter = null, - ITxStorage txStorage = null, + IBlobTxStorage txStorage = null, bool thereIsPriorityContract = false) { specProvider ??= MainnetSpecProvider.Instance; diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 17077fc622b..d90535edfad 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -14,7 +14,7 @@ namespace Nethermind.TxPool; -public class BlobTxStorage : ITxStorage +public class BlobTxStorage : IBlobTxStorage { private static readonly TxDecoder _txDecoder = new(); private static readonly LightTxDecoder _lightTxDecoder = new(); diff --git a/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs new file mode 100644 index 00000000000..a31b9532895 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; + +namespace Nethermind.TxPool; + +public interface IBlobTxStorage : ITxStorage +{ + bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions); + void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions); + void DeleteBlobTransactionsFromBlock(long blockNumber); +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 81a2a8b9f18..251d723b9e0 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -15,7 +15,4 @@ public interface ITxStorage IEnumerable GetAll(); void Add(Transaction transaction); void Delete(in ValueHash256 hash, in UInt256 timestamp); - bool TryGetBlobTransactionsFromBlock(long blockNumber, out Transaction[]? blockBlobTransactions); - void AddBlobTransactionsFromBlock(long blockNumber, IList blockBlobTransactions); - void DeleteBlobTransactionsFromBlock(long blockNumber); } diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index 2a0b8bda575..6eeaf9df353 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -10,7 +10,7 @@ namespace Nethermind.TxPool; -public class NullBlobTxStorage : ITxStorage +public class NullBlobTxStorage : IBlobTxStorage { public static NullBlobTxStorage Instance { get; } = new(); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index c1483e85598..d65e6a823d1 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -45,7 +45,7 @@ public class TxPool : ITxPool, IDisposable private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; private readonly IEthereumEcdsa _ecdsa; - private readonly ITxStorage _blobTxStorage; + private readonly IBlobTxStorage _blobTxStorage; private readonly IChainHeadInfoProvider _headInfo; private readonly ITxPoolConfig _txPoolConfig; private readonly bool _blobReorgsSupportEnabled; @@ -80,7 +80,7 @@ public class TxPool : ITxPool, IDisposable /// /// public TxPool(IEthereumEcdsa ecdsa, - ITxStorage blobTxStorage, + IBlobTxStorage blobTxStorage, IChainHeadInfoProvider chainHeadInfoProvider, ITxPoolConfig txPoolConfig, ITxValidator validator, From c412e401da0dddc85a84f4edf6cfe74516621b38 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 7 Dec 2023 13:32:53 +0100 Subject: [PATCH 25/41] use batch when cleaning processed txs db --- .../ProcessedTransactionsDbCleaner.cs | 15 ++++++++++----- .../SnapSync/SnapProvider.cs | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index c78c8f5cb45..dd8a23991d6 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; +using Nethermind.Core.Collections; using Nethermind.Core.Extensions; using Nethermind.Db; using Nethermind.Logging; @@ -40,19 +41,23 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) { try { - List blocksInDb = new(); + ArrayPoolList blocksInDb = new(); foreach (byte[] key in _processedTxsDb.GetAllKeys()) { blocksInDb.Add(key.ToLongFromBigEndianByteArrayWithoutLeadingZeros()); } if (_logger.IsTrace) _logger.Trace($"There are processed blob txs from {blocksInDb.Count} blocks in ProcessedTxs db"); - foreach (long blockNumber in blocksInDb) + + using (IWriteBatch writeBatch = _processedTxsDb.StartWriteBatch()) { - if (newlyFinalizedBlockNumber >= blockNumber) + foreach (long blockNumber in blocksInDb) { - if (_logger.IsTrace) _logger.Trace($"Cleaning processed blob txs from block {blockNumber}"); - _processedTxsDb.Delete(blockNumber); + if (newlyFinalizedBlockNumber >= blockNumber) + { + if (_logger.IsTrace) _logger.Trace($"Cleaning processed blob txs from block {blockNumber}"); + writeBatch.Delete(blockNumber); + } } } if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); diff --git a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs index 98d3a0cebf6..d2f51406c0e 100644 --- a/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs +++ b/src/Nethermind/Nethermind.Synchronization/SnapSync/SnapProvider.cs @@ -273,7 +273,7 @@ public void AddCodes(ValueHash256[] requestedHashes, byte[][] codes) { HashSet set = requestedHashes.ToHashSet(); - using (IWriteBatch writeWriteBatch = _dbProvider.CodeDb.StartWriteBatch()) + using (IWriteBatch writeBatch = _dbProvider.CodeDb.StartWriteBatch()) { for (int i = 0; i < codes.Length; i++) { @@ -283,7 +283,7 @@ public void AddCodes(ValueHash256[] requestedHashes, byte[][] codes) if (set.Remove(codeHash)) { Interlocked.Add(ref Metrics.SnapStateSynced, code.Length); - writeWriteBatch[codeHash.Bytes] = code; + writeBatch[codeHash.Bytes] = code; } } } From 62ae09d233a603087e018cb358509b8d06374d4c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 8 Nov 2023 11:55:21 +0100 Subject: [PATCH 26/41] fix --- .../Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index dd8a23991d6..8b20ae83046 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -41,7 +41,7 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) { try { - ArrayPoolList blocksInDb = new(); + List blocksInDb = new(); foreach (byte[] key in _processedTxsDb.GetAllKeys()) { blocksInDb.Add(key.ToLongFromBigEndianByteArrayWithoutLeadingZeros()); From 0e201f6a366797f75c36882ee3c5d2e6b93e05ce Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 7 Dec 2023 15:51:04 +0100 Subject: [PATCH 27/41] fix file encoding --- src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs index a31b9532895..7adb61d7c05 100644 --- a/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/IBlobTxStorage.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; From 5e86fbad40058dbd932565f90b83df9ebd1e0aeb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 8 Dec 2023 10:46:11 +0100 Subject: [PATCH 28/41] fix init when blob support or reorgs disabled --- src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index e7ef997049d..13b78c8cfc8 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -29,8 +29,8 @@ using Nethermind.Merge.Plugin.InvalidChainTracker; using Nethermind.Merge.Plugin.Synchronization; using Nethermind.Synchronization.ParallelSync; -using Nethermind.Synchronization.Reporting; using Nethermind.Trie.Pruning; +using Nethermind.TxPool; namespace Nethermind.Merge.Plugin; @@ -41,6 +41,7 @@ public partial class MergePlugin : IConsensusWrapperPlugin, ISynchronizationPlug protected IMergeConfig _mergeConfig = null!; private ISyncConfig _syncConfig = null!; protected IBlocksConfig _blocksConfig = null!; + protected ITxPoolConfig _txPoolConfig = null!; protected IPoSSwitcher _poSSwitcher = NoPoS.Instance; private IBeaconPivot? _beaconPivot; private BeaconSync? _beaconSync; @@ -67,6 +68,7 @@ public virtual Task Init(INethermindApi nethermindApi) _mergeConfig = nethermindApi.Config(); _syncConfig = nethermindApi.Config(); _blocksConfig = nethermindApi.Config(); + _txPoolConfig = nethermindApi.Config(); MigrateSecondsPerSlot(_blocksConfig, _mergeConfig); @@ -99,8 +101,11 @@ public virtual Task Init(INethermindApi nethermindApi) _api.PoSSwitcher = _poSSwitcher; _api.DisposeStack.Push(_invalidChainTracker); _blockFinalizationManager = new ManualBlockFinalizationManager(); - ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); - _api.DisposeStack.Push(processedTransactionsDbCleaner); + if (_txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true, BlobReorgsSupportEnabled: true }) + { + ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); + _api.DisposeStack.Push(processedTransactionsDbCleaner); + } _api.RewardCalculatorSource = new MergeRewardCalculatorSource( _api.RewardCalculatorSource ?? NoBlockRewards.Instance, _poSSwitcher); From 2d206cba6528e583cd6947fcae180f67c69ea872 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Dec 2023 16:11:33 +0100 Subject: [PATCH 29/41] fix signing test txs --- src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs b/src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs index eb32dbe4c3b..48eeb1d07be 100644 --- a/src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs +++ b/src/Nethermind/Nethermind.Crypto/EthereumEcdsa.cs @@ -41,16 +41,14 @@ public void Sign(PrivateKey privateKey, Transaction tx, bool isEip155Enabled) _logger.Debug( $"Signing transaction {tx.SenderAddress} -> {tx.To} ({tx.Value}) with data of length {tx.Data?.Length}"); - //Keccak hash = Keccak.Compute(Bytes.Concat((byte)tx.Type, Rlp.Encode(tx, true, isEip155Enabled, _chainIdValue).Bytes)); - - Hash256 hash = Keccak.Compute(Rlp.Encode(tx, true, isEip155Enabled, _chainIdValue).Bytes); - tx.Signature = Sign(privateKey, hash); - if (tx.Type != TxType.Legacy) { tx.ChainId = _chainIdValue; } + Hash256 hash = Keccak.Compute(Rlp.Encode(tx, true, isEip155Enabled, _chainIdValue).Bytes); + tx.Signature = Sign(privateKey, hash); + if (tx.Type == TxType.Legacy && isEip155Enabled) { tx.Signature.V = tx.Signature.V + 8 + 2 * _chainIdValue; From 996a62d47f6eac87b217871d30cd61b26e5c02a8 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Dec 2023 16:13:09 +0100 Subject: [PATCH 30/41] add regression test --- .../Crypto/EthereumEcdsaTests.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Nethermind/Nethermind.Core.Test/Crypto/EthereumEcdsaTests.cs b/src/Nethermind/Nethermind.Core.Test/Crypto/EthereumEcdsaTests.cs index 07728abe408..aa561677ad7 100644 --- a/src/Nethermind/Nethermind.Core.Test/Crypto/EthereumEcdsaTests.cs +++ b/src/Nethermind/Nethermind.Core.Test/Crypto/EthereumEcdsaTests.cs @@ -38,6 +38,18 @@ public void Signature_test_sepolia(bool eip155) Assert.That(address, Is.EqualTo(key.Address)); } + [TestCase(true)] + [TestCase(false)] + public void Signature_test_sepolia_1559(bool eip155) + { + EthereumEcdsa ecdsa = new(BlockchainIds.Sepolia, LimboLogs.Instance); + PrivateKey key = Build.A.PrivateKey.TestObject; + Transaction tx = Build.A.Transaction.WithType(TxType.EIP1559).TestObject; + ecdsa.Sign(key, tx, eip155); + Address? address = ecdsa.RecoverAddress(tx); + Assert.That(address, Is.EqualTo(key.Address)); + } + [TestCase(true)] [TestCase(false)] public void Signature_test_olympic(bool isEip155Enabled) From 0f4e1f9cbc33f73086f2194e856ea544a2c68c45 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Dec 2023 16:13:52 +0100 Subject: [PATCH 31/41] fix tests --- .../Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs | 4 ++-- .../EngineModuleTests.PayloadProduction.cs | 4 ++-- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs index 2b34ff628fc..1505a222793 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/Eth/EthRpcModuleTests.cs @@ -107,7 +107,7 @@ public async Task Eth_pending_transactions_1559_tx() { using Context ctx = await Context.CreateWithLondonEnabled(); ctx.Test.AddTransactions(Build.A.Transaction.WithMaxPriorityFeePerGas(6.GWei()).WithMaxFeePerGas(11.GWei()).WithType(TxType.EIP1559).SignedAndResolved(TestItem.PrivateKeyC).TestObject); - const string addedTx = "\"hash\":\"0xc668c8940b7416fe06db0dac853210d6d64fb2e9528c439c135a53106517fca6\",\"nonce\":\"0x0\",\"blockHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"blockNumber\":null,\"transactionIndex\":null,\"from\":\"0x76e68a8696537e4141926f3e528733af9e237d69\",\"to\":\"0x0000000000000000000000000000000000000000\",\"value\":\"0x1\",\"gasPrice\":\"0x28fa6ae00\",\"maxPriorityFeePerGas\":\"0x165a0bc00\",\"maxFeePerGas\":\"0x28fa6ae00\",\"gas\":\"0x5208\",\"input\":\"0x\",\"chainId\":\"0x1\",\"type\":\"0x2\",\"accessList\":[],\"v\":\"0x1\",\"s\":\"0x24e1404423c47d5c5fd9e0b6205811eaa3052f9acdb91a9c08821c2b7a0db1a4\",\"r\":\"0x408e34747109a32b924c61acb879d628505dbd0dcab15a3b1e3a4cfd589b65d2\",\"yParity\":\"0x1\""; + const string addedTx = "\"hash\":\"0x7544f95c68426cb8a8a5a54889c60849ed96ff317835beb63b4d745cbc078cec\",\"nonce\":\"0x0\",\"blockHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"blockNumber\":null,\"transactionIndex\":null,\"from\":\"0x76e68a8696537e4141926f3e528733af9e237d69\",\"to\":\"0x0000000000000000000000000000000000000000\",\"value\":\"0x1\",\"gasPrice\":\"0x28fa6ae00\",\"maxPriorityFeePerGas\":\"0x165a0bc00\",\"maxFeePerGas\":\"0x28fa6ae00\",\"gas\":\"0x5208\",\"input\":\"0x\",\"chainId\":\"0x1\",\"type\":\"0x2\",\"accessList\":[],\"v\":\"0x0\",\"s\":\"0x606b869eab1c9d01ff462f887826cb8f349ea8f1b59d0635ae77155b3b84ad86\",\"r\":\"0x63b08cc0a06c88fb1dd79f273736b3463af12c6754f9df764aa222d2693a5d43\",\"yParity\":\"0x0\""; string serialized = await ctx.Test.TestEthRpc("eth_pendingTransactions"); serialized.Contains(addedTx).Should().BeTrue(); } @@ -117,7 +117,7 @@ public async Task Eth_pending_transactions_2930_tx() { using Context ctx = await Context.CreateWithLondonEnabled(); ctx.Test.AddTransactions(Build.A.Transaction.WithMaxPriorityFeePerGas(6.GWei()).WithMaxFeePerGas(11.GWei()).WithType(TxType.AccessList).SignedAndResolved(TestItem.PrivateKeyC).TestObject); - const string addedTx = "\"hash\":\"0xa296c4cf8ece2d7788e4a71125133dfd8025fc35cc5ffa3e283bc62b027cf512\",\"nonce\":\"0x0\",\"blockHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"blockNumber\":null,\"transactionIndex\":null,\"from\":\"0x76e68a8696537e4141926f3e528733af9e237d69\",\"to\":\"0x0000000000000000000000000000000000000000\",\"value\":\"0x1\",\"gasPrice\":\"0x165a0bc00\",\"gas\":\"0x5208\",\"input\":\"0x\",\"chainId\":\"0x1\",\"type\":\"0x1\",\"accessList\":[],\"v\":\"0x0\",\"s\":\"0x2b4cbea82cc417cdf510fb5fb0613a2881f2b8a76cd6cce6a5f77872f5124b44\",\"r\":\"0x925ede2e48031b060e6c4a0c7184eb58e37a19b41a3ba15a2d9767c0c41f6d76\",\"yParity\":\"0x0\""; + const string addedTx = "\"hash\":\"0x4eabe360dc515aadc8e35f75b23803bb86e7186ebf2e58412555b3d0c7750dcc\",\"nonce\":\"0x0\",\"blockHash\":\"0x0000000000000000000000000000000000000000000000000000000000000000\",\"blockNumber\":null,\"transactionIndex\":null,\"from\":\"0x76e68a8696537e4141926f3e528733af9e237d69\",\"to\":\"0x0000000000000000000000000000000000000000\",\"value\":\"0x1\",\"gasPrice\":\"0x165a0bc00\",\"gas\":\"0x5208\",\"input\":\"0x\",\"chainId\":\"0x1\",\"type\":\"0x1\",\"accessList\":[],\"v\":\"0x0\",\"s\":\"0x27e3dde7b07d6d6b50e0d11b29085036e9c8adc12dea52f6f07dd7a0551ff22a\",\"r\":\"0x619cb31fd4aa1c38ae36b31c5d8310f74d9f8ddd94389db91a68deb26737f2dc\",\"yParity\":\"0x0\""; string serialized = await ctx.Test.TestEthRpc("eth_pendingTransactions"); serialized.Contains(addedTx).Should().BeTrue(); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs index 86fe1d35ade..85a569bc94d 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.PayloadProduction.cs @@ -231,7 +231,7 @@ public async Task getPayload_correctlyEncodeTransactions() result = new ExecutionPayload { BaseFeePerGas = 0, - BlockHash = new("0x5fd61518405272d77fd6cdc8a824a109d75343e32024ee4f6769408454b1823d"), + BlockHash = new("0x28012c3a37c85b37f9dc6db2d874f9c92b5d8d4bb784177c5309a0c6d7af6ef4"), BlockNumber = 0, ExtraData = Bytes.FromHexString("0x010203"), FeeRecipient = Address.Zero, @@ -250,7 +250,7 @@ public async Task getPayload_correctlyEncodeTransactions() Bytes.FromHexString( "0xf85f800182520894475674cb523a0a2736b7f7534390288fce16982c018025a0634db2f18f24d740be29e03dd217eea5757ed7422680429bdd458c582721b6c2a02f0fa83931c9a99d3448a46b922261447d6a41d8a58992b5596089d15d521102"), Bytes.FromHexString( - "0x02f8620180011482520894475674cb523a0a2736b7f7534390288fce16982c0180c001a0033e85439a128c42f2ba47ca278f1375ef211e61750018ff21bcd9750d1893f2a04ee981fe5261f8853f95c865232ffdab009abcc7858ca051fb624c49744bf18d") + "0x02f8620180011482520894475674cb523a0a2736b7f7534390288fce16982c0180c001a0db002b398e038bc919b316a214154aa6d9d5e404cb201aa8a151efb92f9fdbbda07bee8ea6915ed54bb07af4cd69b201548fe9aac699978e5c444405dc49f55a36") }, }, id = 67 diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index beea6b95e6c..478055f47fa 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -477,7 +477,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) [TestCase(0, 97)] [TestCase(1, 131320)] - [TestCase(2, 262529)] + [TestCase(2, 262530)] [TestCase(3, 393737)] [TestCase(4, 524944)] [TestCase(5, 656152)] From 089ff10e2692a4d4704ae7672726215f29b07035 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Dec 2023 16:33:20 +0100 Subject: [PATCH 32/41] one more regression test --- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 478055f47fa..f19a671aa60 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -490,5 +490,18 @@ public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int ex .TestObject; blobTx.GetLength().Should().Be(expectedLength); } + + [Test] + public void RecoverAddress_should_work_correctly() + { + Transaction tx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _ethereumEcdsa.RecoverAddress(tx).Should().Be(tx.SenderAddress); + } } } From 04bdd240619e759a07e06eaaf763b39280516cc5 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Dec 2023 16:57:32 +0100 Subject: [PATCH 33/41] post-merge fix --- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index f6a01fe06d1..4ba704d7c54 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -17,7 +17,6 @@ namespace Nethermind.TxPool; public class BlobTxStorage : IBlobTxStorage { private static readonly TxDecoder _txDecoder = new(); - private static readonly LightTxDecoder _lightTxDecoder = new(); private readonly IDb _fullBlobTxsDb; private readonly IDb _lightBlobTxsDb; private readonly IDb _processedBlobTxsDb; From b509665b73fbe7047d4c9cd6559fa000df70c54e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Dec 2023 15:21:02 +0100 Subject: [PATCH 34/41] add test for readding reorganized blob txs --- .../TxPoolTests.Blobs.cs | 83 +++++++++++++++---- .../Nethermind.TxPool.Test/TxPoolTests.cs | 8 +- 2 files changed, 74 insertions(+), 17 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index c7dc77d7e1f..e3ce0fadec8 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -493,18 +493,15 @@ public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int ex } [Test] - public async Task should_add_processed_txs_to_db() + public void RecoverAddress_should_work_correctly() { - Transaction GetTx(PrivateKey sender) - { - return Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, sender).TestObject; - } + Transaction tx = GetTx(TestItem.PrivateKeyA); + _ethereumEcdsa.RecoverAddress(tx).Should().Be(tx.SenderAddress); + } + [Test] + public async Task should_add_processed_txs_to_db() + { const long blockNumber = 358; BlobTxStorage blobTxStorage = new(); @@ -549,16 +546,72 @@ Transaction GetTx(PrivateKey sender) } [Test] - public void RecoverAddress_should_work_correctly() + public async Task should_bring_back_reorganized_blob_txs() { - Transaction tx = Build.A.Transaction + const long blockNumber = 358; + + BlobTxStorage blobTxStorage = new(); + ITxPoolConfig txPoolConfig = new TxPoolConfig() + { + Size = 128, + BlobSupportEnabled = true, + PersistentBlobStorageEnabled = true, + BlobReorgsSupportEnabled = true + }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressC, UInt256.MaxValue); + + Transaction[] txsA = { GetTx(TestItem.PrivateKeyA), GetTx(TestItem.PrivateKeyB) }; + Transaction[] txsB = { GetTx(TestItem.PrivateKeyC) }; + + _txPool.SubmitTx(txsA[0], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(txsA[1], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(txsB[0], TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(txsA.Length + txsB.Length); + + // adding block A + Block blockA = Build.A.Block.WithNumber(blockNumber).WithTransactions(txsA).TestObject; + await RaiseBlockAddedToMainAndWaitForTransactions(txsA.Length, blockA); + + _txPool.GetPendingBlobTransactionsCount().Should().Be(txsB.Length); + _txPool.TryGetPendingBlobTransaction(txsA[0].Hash!, out _).Should().BeFalse(); + _txPool.TryGetPendingBlobTransaction(txsA[1].Hash!, out _).Should().BeFalse(); + _txPool.TryGetPendingBlobTransaction(txsB[0].Hash!, out _).Should().BeTrue(); + + // reorganized from block A to block B + Block blockB = Build.A.Block.WithNumber(blockNumber).WithTransactions(txsB).TestObject; + await RaiseBlockAddedToMainAndWaitForTransactions(txsB.Length + txsA.Length, blockB, blockA); + + // tx from block B should be removed from blob pool + _txPool.TryGetPendingBlobTransaction(txsB[0].Hash!, out _).Should().BeFalse(); + + // blob txs from reorganized blockA should be readded to blob pool + _txPool.GetPendingBlobTransactionsCount().Should().Be(txsA.Length); + _txPool.TryGetPendingBlobTransaction(txsA[0].Hash!, out Transaction tx1).Should().BeTrue(); + _txPool.TryGetPendingBlobTransaction(txsA[1].Hash!, out Transaction tx2).Should().BeTrue(); + + tx1.Should().BeEquivalentTo(txsA[0], options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex + + tx2.Should().BeEquivalentTo(txsA[1], options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex + } + + private Transaction GetTx(PrivateKey sender) + { + return Build.A.Transaction .WithShardBlobTxTypeAndFields() .WithMaxFeePerGas(1.GWei()) .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _ethereumEcdsa.RecoverAddress(tx).Should().Be(tx.SenderAddress); + .SignedAndResolved(_ethereumEcdsa, sender).TestObject; } } } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index b897e23c2f5..e2555cd1bd1 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1803,11 +1803,15 @@ private Transaction GetTransaction(UInt256 nonce, long gasLimit, UInt256 gasPric .SignedAndResolved(_ethereumEcdsa, privateKey) .TestObject; - private async Task RaiseBlockAddedToMainAndWaitForTransactions(int txCount, Block block = null) + private async Task RaiseBlockAddedToMainAndWaitForTransactions(int txCount, Block block = null, Block previousBlock = null) { + BlockReplacementEventArgs blockReplacementEventArgs = previousBlock is null + ? new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject) + : new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject, previousBlock); + SemaphoreSlim semaphoreSlim = new(0, txCount); _txPool.NewPending += (o, e) => semaphoreSlim.Release(); - _blockTree.BlockAddedToMain += Raise.EventWith(new BlockReplacementEventArgs(block ?? Build.A.Block.TestObject)); + _blockTree.BlockAddedToMain += Raise.EventWith(blockReplacementEventArgs); for (int i = 0; i < txCount; i++) { await semaphoreSlim.WaitAsync(10); From 28ce30fe019a28d7778b4efc5b6a32bb8a3b8feb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 20 Dec 2023 10:30:08 +0100 Subject: [PATCH 35/41] improve test --- .../ProcessedTransactionsDbCleanerTests.cs | 5 ++++- .../ProcessedTransactionsDbCleaner.cs | 11 +++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs index 09612a29235..c07a7cc9b95 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -54,7 +54,10 @@ Transaction GetTx(PrivateKey sender) new FinalizeEventArgs(Build.A.BlockHeader.TestObject, Build.A.BlockHeader.WithNumber(finalizedBlock).TestObject)); - Thread.Sleep(100); + while (dbCleaner.LastFinalizedBlock == 0L) + { + Thread.Sleep(10); + } blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out returnedTxs).Should().Be(blockOfTxs > finalizedBlock); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index 8b20ae83046..21386b9b14e 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; -using Nethermind.Core.Collections; using Nethermind.Core.Extensions; using Nethermind.Db; using Nethermind.Logging; @@ -18,7 +17,7 @@ public class ProcessedTransactionsDbCleaner : IDisposable private readonly IBlockFinalizationManager _finalizationManager; private readonly IDb _processedTxsDb; private readonly ILogger _logger; - private long _lastFinalizedBlock = 0; + internal long LastFinalizedBlock = 0; public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) { @@ -31,7 +30,7 @@ public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationMana private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) { - if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > _lastFinalizedBlock) + if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > LastFinalizedBlock) { Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); } @@ -60,13 +59,13 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) } } } - if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); + if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {LastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); - _lastFinalizedBlock = newlyFinalizedBlockNumber; + LastFinalizedBlock = newlyFinalizedBlockNumber; } catch (Exception exception) { - if (_logger.IsError) _logger.Error($"Couldn't correctly clean db with processed transactions. Newly finalized block {newlyFinalizedBlockNumber}, last finalized block: {_lastFinalizedBlock}", exception); + if (_logger.IsError) _logger.Error($"Couldn't correctly clean db with processed transactions. Newly finalized block {newlyFinalizedBlockNumber}, last finalized block: {LastFinalizedBlock}", exception); } } From e3af32576b90fb14db4d5e294696f3c2ad0ac95b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 20 Dec 2023 10:42:51 +0100 Subject: [PATCH 36/41] one more check in test --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index e3ce0fadec8..63ac0733bdd 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -587,8 +587,13 @@ public async Task should_bring_back_reorganized_blob_txs() Block blockB = Build.A.Block.WithNumber(blockNumber).WithTransactions(txsB).TestObject; await RaiseBlockAddedToMainAndWaitForTransactions(txsB.Length + txsA.Length, blockB, blockA); - // tx from block B should be removed from blob pool + // tx from block B should be removed from blob pool, but present in processed txs db _txPool.TryGetPendingBlobTransaction(txsB[0].Hash!, out _).Should().BeFalse(); + blobTxStorage.TryGetBlobTransactionsFromBlock(blockNumber, out Transaction[] blockBTxs).Should().BeTrue(); + txsB.Should().BeEquivalentTo(blockBTxs, options => options + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex) // ...as well as PoolIndex + .Excluding(t => t.SenderAddress)); // sender is recovered later, it is not returned from db // blob txs from reorganized blockA should be readded to blob pool _txPool.GetPendingBlobTransactionsCount().Should().Be(txsA.Length); From 129241166a1bc5cc127f3517670dff4ecf5b759e Mon Sep 17 00:00:00 2001 From: "lukasz.rozmej" Date: Thu, 21 Dec 2023 16:14:30 +0100 Subject: [PATCH 37/41] Remove unnecessary list Sleep -> Task in test --- .../ProcessedTransactionsDbCleanerTests.cs | 8 +++----- .../ProcessedTransactionsDbCleaner.cs | 15 +++++---------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs index c07a7cc9b95..b8622cccc25 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Threading; +using System.Threading.Tasks; using FluentAssertions; using Nethermind.Blockchain; using Nethermind.Core; @@ -26,7 +27,7 @@ public class ProcessedTransactionsDbCleanerTests private readonly ISpecProvider _specProvider = MainnetSpecProvider.Instance; [Test] - public void should_remove_processed_txs_from_db_after_finalization([Values(0, 1, 42, 358)] long blockOfTxs, [Values(1, 42, 358)] long finalizedBlock) + public async Task should_remove_processed_txs_from_db_after_finalization([Values(0, 1, 42, 358)] long blockOfTxs, [Values(1, 42, 358)] long finalizedBlock) { Transaction GetTx(PrivateKey sender) { @@ -54,10 +55,7 @@ Transaction GetTx(PrivateKey sender) new FinalizeEventArgs(Build.A.BlockHeader.TestObject, Build.A.BlockHeader.WithNumber(finalizedBlock).TestObject)); - while (dbCleaner.LastFinalizedBlock == 0L) - { - Thread.Sleep(10); - } + await dbCleaner.CleaningTask; blobTxStorage.TryGetBlobTransactionsFromBlock(blockOfTxs, out returnedTxs).Should().Be(blockOfTxs > finalizedBlock); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index 21386b9b14e..9771e66e50c 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -18,6 +18,7 @@ public class ProcessedTransactionsDbCleaner : IDisposable private readonly IDb _processedTxsDb; private readonly ILogger _logger; internal long LastFinalizedBlock = 0; + public Task CleaningTask { get; private set; } = Task.CompletedTask; public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) { @@ -32,7 +33,7 @@ private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) { if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > LastFinalizedBlock) { - Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); + CleaningTask = Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); } } @@ -40,18 +41,11 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) { try { - List blocksInDb = new(); - foreach (byte[] key in _processedTxsDb.GetAllKeys()) - { - blocksInDb.Add(key.ToLongFromBigEndianByteArrayWithoutLeadingZeros()); - } - - if (_logger.IsTrace) _logger.Trace($"There are processed blob txs from {blocksInDb.Count} blocks in ProcessedTxs db"); - using (IWriteBatch writeBatch = _processedTxsDb.StartWriteBatch()) { - foreach (long blockNumber in blocksInDb) + foreach (byte[] key in _processedTxsDb.GetAllKeys()) { + long blockNumber = key.ToLongFromBigEndianByteArrayWithoutLeadingZeros(); if (newlyFinalizedBlockNumber >= blockNumber) { if (_logger.IsTrace) _logger.Trace($"Cleaning processed blob txs from block {blockNumber}"); @@ -59,6 +53,7 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) } } } + if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {LastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); LastFinalizedBlock = newlyFinalizedBlockNumber; From 860c9ab13080da02a5d0ed7f0b476eb5048c96d2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 21 Dec 2023 16:56:16 +0100 Subject: [PATCH 38/41] cosmetic --- .../ProcessedTransactionsDbCleaner.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs index 9771e66e50c..7babdf12c72 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/ProcessedTransactionsDbCleaner.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.Threading.Tasks; using Nethermind.Blockchain; using Nethermind.Core; @@ -17,7 +16,7 @@ public class ProcessedTransactionsDbCleaner : IDisposable private readonly IBlockFinalizationManager _finalizationManager; private readonly IDb _processedTxsDb; private readonly ILogger _logger; - internal long LastFinalizedBlock = 0; + private long _lastFinalizedBlock = 0; public Task CleaningTask { get; private set; } = Task.CompletedTask; public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationManager, IDb processedTxsDb, ILogManager logManager) @@ -31,7 +30,7 @@ public ProcessedTransactionsDbCleaner(IBlockFinalizationManager finalizationMana private void OnBlocksFinalized(object? sender, FinalizeEventArgs e) { - if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > LastFinalizedBlock) + if (e.FinalizedBlocks.Count > 0 && e.FinalizedBlocks[0].Number > _lastFinalizedBlock) { CleaningTask = Task.Run(() => CleanProcessedTransactionsDb(e.FinalizedBlocks[0].Number)); } @@ -54,13 +53,13 @@ private void CleanProcessedTransactionsDb(long newlyFinalizedBlockNumber) } } - if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {LastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); + if (_logger.IsDebug) _logger.Debug($"Cleaned processed blob txs from block {_lastFinalizedBlock} to block {newlyFinalizedBlockNumber}"); - LastFinalizedBlock = newlyFinalizedBlockNumber; + _lastFinalizedBlock = newlyFinalizedBlockNumber; } catch (Exception exception) { - if (_logger.IsError) _logger.Error($"Couldn't correctly clean db with processed transactions. Newly finalized block {newlyFinalizedBlockNumber}, last finalized block: {LastFinalizedBlock}", exception); + if (_logger.IsError) _logger.Error($"Couldn't correctly clean db with processed transactions. Newly finalized block {newlyFinalizedBlockNumber}, last finalized block: {_lastFinalizedBlock}", exception); } } From 045c26c0c6810bb62efda548801ece1f7824c237 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 22 Dec 2023 15:58:49 +0100 Subject: [PATCH 39/41] compress 3 bool flags to 1 enum --- .../Blockchain/TestBlockchain.cs | 2 +- .../Nethermind.Init/Steps/InitDatabase.cs | 2 +- .../ProcessedTransactionsDbCleanerTests.cs | 1 - .../Nethermind.Merge.Plugin/MergePlugin.cs | 2 +- .../Subprotocols/Eth/PooledTxsRequestor.cs | 2 +- .../TxPoolTests.Blobs.cs | 50 ++++++++----------- .../Nethermind.TxPool/BlobsSupportMode.cs | 38 ++++++++++++++ .../Filters/NotSupportedTxFilter.cs | 2 +- .../Nethermind.TxPool/ITxPoolConfig.cs | 16 +++--- src/Nethermind/Nethermind.TxPool/TxPool.cs | 6 +-- .../Nethermind.TxPool/TxPoolConfig.cs | 4 +- 11 files changed, 76 insertions(+), 49 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index e23c0705e0d..4c83df1211f 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -283,7 +283,7 @@ protected virtual TxPool.TxPool CreateTxPool() => EthereumEcdsa, new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(SpecProvider), BlockTree, ReadOnlyState), - new TxPoolConfig() { BlobSupportEnabled = true }, + new TxPoolConfig() { BlobsSupport = BlobsSupportMode.InMemory }, new TxValidator(SpecProvider.ChainId), LogManager, TransactionComparerProvider.GetDefaultComparer()); diff --git a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs index 1a970d399ba..ac870740367 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs @@ -46,7 +46,7 @@ public virtual async Task Execute(CancellationToken _) try { bool useReceiptsDb = initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync; - bool useBlobsDb = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true }; + bool useBlobsDb = txPoolConfig.BlobsSupport.IsPersistentStorage(); InitDbApi(initConfig, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync); StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem); await dbInitializer.InitStandardDbsAsync(useReceiptsDb, useBlobsDb); diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs index b8622cccc25..8e0b754b1a8 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/ProcessedTransactionsDbCleanerTests.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Blockchain; diff --git a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs index 2c2ed351bc1..2213acad4e9 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs @@ -101,7 +101,7 @@ public virtual Task Init(INethermindApi nethermindApi) _api.PoSSwitcher = _poSSwitcher; _api.DisposeStack.Push(_invalidChainTracker); _blockFinalizationManager = new ManualBlockFinalizationManager(); - if (_txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true, BlobReorgsSupportEnabled: true }) + if (_txPoolConfig.BlobsSupport.SupportsReorgs()) { ProcessedTransactionsDbCleaner processedTransactionsDbCleaner = new(_blockFinalizationManager, _api.DbProvider.BlobTransactionsDb.GetColumnDb(BlobTxsColumns.ProcessedTxs), _api.LogManager); _api.DisposeStack.Push(processedTransactionsDbCleaner); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs index 3aa898824dc..ae3bd356a7f 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs @@ -94,7 +94,7 @@ public void RequestTransactionsEth68(Action +/// Defines blobs support mode. +/// +public enum BlobsSupportMode +{ + /// + /// No support for blob transactions. + /// + Disabled, + + /// + /// Blob transactions stored only in memory + /// + InMemory, + + /// + /// Blob transactions stored in db. + /// + Storage, + + /// + /// Blob transactions stored in db with support for restoring reorganized blob transactions to blob pool. + /// + StorageWithReorgs +} + +public static class BlobsSupportModeExtensions +{ + public static bool IsPersistentStorage(this BlobsSupportMode mode) => mode is BlobsSupportMode.Storage or BlobsSupportMode.StorageWithReorgs; + public static bool IsEnabled(this BlobsSupportMode mode) => mode is not BlobsSupportMode.Disabled; + public static bool IsDisabled(this BlobsSupportMode mode) => mode is BlobsSupportMode.Disabled; + public static bool SupportsReorgs(this BlobsSupportMode mode) => mode is BlobsSupportMode.StorageWithReorgs; +} diff --git a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs index 2f2e264e3d6..a91d89e4978 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs @@ -22,7 +22,7 @@ public NotSupportedTxFilter(ITxPoolConfig txPoolConfig, ILogger logger) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) { - if (!_txPoolConfig.BlobSupportEnabled && tx.SupportsBlobs) + if (_txPoolConfig.BlobsSupport.IsDisabled() && tx.SupportsBlobs) { Metrics.PendingTransactionsNotSupportedTxType++; if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, blob transactions are not supported."); diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 9f6a1cd20da..c6e630822ce 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -16,11 +16,16 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "2048", Description = "The max number of transactions held in the mempool (the more transactions in the mempool, the more memory used).")] int Size { get; set; } - [ConfigItem(DefaultValue = "false", Description = "Whether to enable blob transactions.")] - bool BlobSupportEnabled { get; set; } + [ConfigItem( + Description = """ + Blobs support mode: - [ConfigItem(DefaultValue = "false", Description = "Whether to store blob transactions in the database.")] - bool PersistentBlobStorageEnabled { get; set; } + - `Disabled`: No support for blob transactions + - `InMemory`: Blob transactions stored only in memory + - `Storage`: Blob transactions stored in db + - `StorageWithReorgs`: Blob transactions stored in db with support for restoring reorganized blob transactions to blob pool + """, DefaultValue = "Disabled")] + BlobsSupportMode BlobsSupport { get; set; } [ConfigItem(DefaultValue = "16384", Description = "The max number of full blob transactions stored in the database (increasing the number of transactions in the blob pool also results in higher memory usage). The default value uses max 13GB for 6 blobs where one blob is 2GB (16386 * 128KB).")] int PersistentBlobStorageSize { get; set; } @@ -31,9 +36,6 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "512", Description = "The max number of full blob transactions stored in memory. Used only if persistent storage is disabled.")] int InMemoryBlobPoolSize { get; set; } - [ConfigItem(DefaultValue = "true", Description = "If true, all processed blob transactions would be stored in persistent db until finalization to readd them to the pool in case of chain reorganization")] - bool BlobReorgsSupportEnabled { get; set; } - [ConfigItem(DefaultValue = "0", Description = "The max number of pending transactions per single sender. `0` to lift the limit.")] int MaxPendingTxsPerSender { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 67412420f9f..235f149a88c 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -95,7 +95,7 @@ public TxPool(IEthereumEcdsa ecdsa, _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; - _blobReorgsSupportEnabled = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true, BlobReorgsSupportEnabled: true }; + _blobReorgsSupportEnabled = txPoolConfig.BlobsSupport.SupportsReorgs(); _accounts = _headInfo.AccountStateProvider; _specProvider = _headInfo.SpecProvider; @@ -108,9 +108,9 @@ public TxPool(IEthereumEcdsa ecdsa, _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); - _blobTransactions = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true } + _blobTransactions = txPoolConfig.BlobsSupport.IsPersistentStorage() ? new PersistentBlobTxDistinctSortedPool(blobTxStorage, _txPoolConfig, comparer, logManager) - : new BlobTxDistinctSortedPool(txPoolConfig.BlobSupportEnabled ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); + : new BlobTxDistinctSortedPool(txPoolConfig.BlobsSupport == BlobsSupportMode.InMemory ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); if (_blobTransactions.Count > 0) _blobTransactions.UpdatePool(_accounts, _updateBucket); _headInfo.HeadChanged += OnHeadChange; diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 67c7ac747b0..b2484b33799 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -8,14 +8,12 @@ public class TxPoolConfig : ITxPoolConfig public int PeerNotificationThreshold { get; set; } = 5; public int MinBaseFeeThreshold { get; set; } = 70; public int Size { get; set; } = 2048; - public bool BlobSupportEnabled { get; set; } = false; - public bool PersistentBlobStorageEnabled { get; set; } = false; + public BlobsSupportMode BlobsSupport { get; set; } = BlobsSupportMode.Disabled; public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 13GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); // practical max - something between, but closer to 2GB than 12GB. Geth is limiting it to 10GB. // every day about 21600 blobs will be included (7200 blocks per day * 3 blob target) public int BlobCacheSize { get; set; } = 256; public int InMemoryBlobPoolSize { get; set; } = 512; // it is used when persistent pool is disabled - public bool BlobReorgsSupportEnabled { get; set; } = true; public int MaxPendingTxsPerSender { get; set; } = 0; public int MaxPendingBlobTxsPerSender { get; set; } = 16; public int HashCacheSize { get; set; } = 512 * 1024; From 286b49a266331ddf2c00dc0aff7f74e8e19564e2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 20 Dec 2023 11:04:22 +0100 Subject: [PATCH 40/41] fix file encoding --- src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs b/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs index dcb4673df53..292706d5099 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobsSupportMode.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only namespace Nethermind.TxPool; From c35928dc398fac7532a3b7b1a8c86635d20675d6 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 22 Dec 2023 16:24:07 +0100 Subject: [PATCH 41/41] fix whitespaces --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 2 +- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index cb0a0f0d78b..f4d2b2092ae 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -19,7 +19,7 @@ public partial class TxPoolTests [Test] public void should_reject_blob_tx_if_blobs_not_supported([Values(true, false)] bool isBlobSupportEnabled) { - TxPoolConfig txPoolConfig = new() { BlobsSupport = isBlobSupportEnabled ? BlobsSupportMode.InMemory : BlobsSupportMode.Disabled}; + TxPoolConfig txPoolConfig = new() { BlobsSupport = isBlobSupportEnabled ? BlobsSupportMode.InMemory : BlobsSupportMode.Disabled }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); Transaction tx = Build.A.Transaction diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index c6e630822ce..992c2c8db92 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -25,7 +25,7 @@ public interface ITxPoolConfig : IConfig - `Storage`: Blob transactions stored in db - `StorageWithReorgs`: Blob transactions stored in db with support for restoring reorganized blob transactions to blob pool """, DefaultValue = "Disabled")] - BlobsSupportMode BlobsSupport { get; set; } + BlobsSupportMode BlobsSupport { get; set; } [ConfigItem(DefaultValue = "16384", Description = "The max number of full blob transactions stored in the database (increasing the number of transactions in the blob pool also results in higher memory usage). The default value uses max 13GB for 6 blobs where one blob is 2GB (16386 * 128KB).")] int PersistentBlobStorageSize { get; set; }