From 069807dea7873a857b45cd6c0558895405e29642 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 21 Jul 2023 15:19:53 -0300 Subject: [PATCH 01/23] Introduce filter limits - Filters can include optional limits --- tools/Nethermind.Tools.Kute/Config.cs | 2 +- .../LimitedJsonRpcMethodFilter.cs | 33 +++++++++++++++++++ .../PatternJsonRpcMethodFilter.cs | 19 +++++++---- .../RegexJsonRpcMethodFilter.cs | 18 ++++++++++ tools/Nethermind.Tools.Kute/Program.cs | 21 +++++++----- tools/Nethermind.Tools.Kute/README.md | 10 ++++-- 6 files changed, 86 insertions(+), 17 deletions(-) create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index 87c0876ac21..2890e9bd94d 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -75,7 +75,7 @@ public class Config Separator = ',', Required = false, Default = new string[] { }, - HelpText = "A comma separated List of regexes of methods to be executed" + HelpText = "A comma separated List of regexes of methods to be executed with optional limits" )] public IEnumerable MethodFilters { get; } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs new file mode 100644 index 00000000000..78e4824ea41 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/LimitedJsonRpcMethodFilter.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Tools.Kute.JsonRpcMethodFilter; + +class LimitedJsonRpcMethodFilter : IJsonRpcMethodFilter +{ + private readonly IJsonRpcMethodFilter _filter; + + private int _usagesLeft; + + public LimitedJsonRpcMethodFilter(IJsonRpcMethodFilter filter, int limit) + { + _filter = filter; + _usagesLeft = limit; + } + + public bool ShouldSubmit(string methodName) + { + if (_filter.ShouldSubmit(methodName)) + { + if (_usagesLeft == 0) + { + return false; + } + + _usagesLeft--; + return true; + } + + return false; + } +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs index 324301214a8..a628b3e9245 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/PatternJsonRpcMethodFilter.cs @@ -1,18 +1,25 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Text.RegularExpressions; - namespace Nethermind.Tools.Kute.JsonRpcMethodFilter; -class PatternJsonRpcMethodFilter : IJsonRpcMethodFilter +public class PatternJsonRpcMethodFilter : IJsonRpcMethodFilter { - private readonly Regex _pattern; + private const char PatternSeparator = '='; + private readonly IJsonRpcMethodFilter _filter; public PatternJsonRpcMethodFilter(string pattern) { - _pattern = new Regex(pattern); + var splitted = pattern.Split(PatternSeparator); + + var regex = new RegexJsonRpcMethodFilter(splitted[0]); + _filter = splitted.Length switch + { + 1 => regex, + 2 => new LimitedJsonRpcMethodFilter(regex, int.Parse(splitted[1])), + _ => throw new ArgumentException($"Unexpected pattern: {pattern}"), + }; } - public bool ShouldSubmit(string methodName) => _pattern.IsMatch(methodName); + public bool ShouldSubmit(string methodName) => _filter.ShouldSubmit(methodName); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs new file mode 100644 index 00000000000..07fd79ec4ad --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcMethodFilter/RegexJsonRpcMethodFilter.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.RegularExpressions; + +namespace Nethermind.Tools.Kute.JsonRpcMethodFilter; + +class RegexJsonRpcMethodFilter : IJsonRpcMethodFilter +{ + private readonly Regex _pattern; + + public RegexJsonRpcMethodFilter(string pattern) + { + _pattern = new Regex(pattern); + } + + public bool ShouldSubmit(string methodName) => _pattern.IsMatch(methodName); +} diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index 61f7f37b92f..d2697469f04 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -48,8 +48,11 @@ static IServiceProvider BuildServiceProvider(Config config) collection.AddSingleton>(new FileMessageProvider(config.MessagesFilePath)); collection.AddSingleton, JsonRpcMessageProvider>(); collection.AddSingleton( - new ComposedJsonRpcMethodFilter(config.MethodFilters.Select(pattern => - new PatternJsonRpcMethodFilter(pattern))) + new ComposedJsonRpcMethodFilter( + config.MethodFilters + .Select(pattern => new PatternJsonRpcMethodFilter(pattern)) + .ToList() + ) ); collection.AddSingleton(provider => config.DryRun @@ -77,12 +80,14 @@ static IServiceProvider BuildServiceProvider(Config config) return new NullProgressReporter(); }); collection.AddSingleton(); - collection.AddSingleton(_ => config.MetricsOutputFormatter switch - { - MetricsOutputFormatter.Report => new MetricsTextOutputFormatter(), - MetricsOutputFormatter.Json => new MetricsJsonOutputFormatter(), - _ => throw new ArgumentOutOfRangeException(), - }); + collection.AddSingleton(_ => + config.MetricsOutputFormatter switch + { + MetricsOutputFormatter.Report => new MetricsTextOutputFormatter(), + MetricsOutputFormatter.Json => new MetricsJsonOutputFormatter(), + _ => throw new ArgumentOutOfRangeException(), + } + ); return collection.BuildServiceProvider(); } diff --git a/tools/Nethermind.Tools.Kute/README.md b/tools/Nethermind.Tools.Kute/README.md index 68c3937981b..80379a055d3 100644 --- a/tools/Nethermind.Tools.Kute/README.md +++ b/tools/Nethermind.Tools.Kute/README.md @@ -34,10 +34,16 @@ Some typical usages are as follow: -i /rpc-logs -s keystore/jwt-secret -o Json ``` -### Use a single message file, using only `engine_*` and `eth_*` methods +### Use a single message file, using only `engine_` and `eth_` methods ``` --i /rpc.0 -s keystore/jwt-secret -f engine_*, eth_* +-i /rpc.0 -s keystore/jwt-secret -f engine_, eth_ +``` + +### Use a single message file, using only the first 100 `engine_` methods + +``` +-i /rpc.0 -s keystore/jwt-secret -f engine_=100 ``` ### Connect to a Nethermind Client running in a specific address and TTL From b5cae4f1aa578f8025b8b02aa6b8c97ca1642ea9 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 21 Jul 2023 15:29:43 -0300 Subject: [PATCH 02/23] Report progress at the start - Fixes progress not updating --- tools/Nethermind.Tools.Kute/Application.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index d5d51c8ce4f..4bd01106efe 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -43,6 +43,8 @@ public async Task Run() { await foreach (var (jsonRpc, n) in _msgProvider.Messages.Indexed(startingFrom: 1)) { + _progressReporter.ReportProgress(n); + _metrics.TickMessages(); switch (jsonRpc) @@ -89,8 +91,6 @@ public async Task Run() default: throw new ArgumentOutOfRangeException(nameof(jsonRpc)); } - - _progressReporter.ReportProgress(n); } } From 3f99c36847658e6fc2b70324f2a3f189a16b0216 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 21 Jul 2023 17:03:30 -0300 Subject: [PATCH 03/23] Initial response validator - Verifies that the response does not contain an 'error' field --- tools/Nethermind.Tools.Kute/Application.cs | 10 +++++++++- .../JsonRpcSubmitter/HttpJsonRpcSubmitter.cs | 7 +++++-- .../JsonRpcSubmitter/IJsonRpcSubmitter.cs | 4 +++- .../JsonRpcSubmitter/NullJsonRpcSubmitter.cs | 5 +++-- .../JsonRpcValidator/IJsonRpcValidator.cs | 12 +++++++++++ .../NonErrorJsonRpcValidator.cs | 20 +++++++++++++++++++ .../JsonRpcValidator/NullJsonRpcValidator.cs | 11 ++++++++++ tools/Nethermind.Tools.Kute/Program.cs | 8 +++++++- 8 files changed, 70 insertions(+), 7 deletions(-) create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index 4bd01106efe..aa160af9c13 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -5,6 +5,7 @@ using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.JsonRpcMethodFilter; using Nethermind.Tools.Kute.JsonRpcSubmitter; +using Nethermind.Tools.Kute.JsonRpcValidator; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; @@ -16,6 +17,7 @@ class Application private readonly IMessageProvider _msgProvider; private readonly IJsonRpcSubmitter _submitter; + private readonly IJsonRpcValidator _validator; private readonly IProgressReporter _progressReporter; private readonly IMetricsConsumer _metricsConsumer; private readonly IJsonRpcMethodFilter _methodFilter; @@ -23,6 +25,7 @@ class Application public Application( IMessageProvider msgProvider, IJsonRpcSubmitter submitter, + IJsonRpcValidator validator, IProgressReporter progressReporter, IMetricsConsumer metricsConsumer, IJsonRpcMethodFilter methodFilter @@ -30,6 +33,7 @@ IJsonRpcMethodFilter methodFilter { _msgProvider = msgProvider; _submitter = submitter; + _validator = validator; _progressReporter = progressReporter; _metricsConsumer = metricsConsumer; _methodFilter = methodFilter; @@ -83,7 +87,11 @@ public async Task Run() using (_metrics.TimeMethod(single.MethodName)) { - await _submitter.Submit(single); + var result = await _submitter.Submit(single); + if (_validator.IsInvalid(result)) + { + _metrics.TickFailed(); + } } break; diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs index 9fd1774ac3f..56a28e51168 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/HttpJsonRpcSubmitter.cs @@ -5,6 +5,7 @@ using System.Net.Http.Headers; using System.Net.Mime; using System.Text; +using System.Text.Json; using Nethermind.Tools.Kute.Auth; namespace Nethermind.Tools.Kute.JsonRpcSubmitter; @@ -22,7 +23,7 @@ public HttpJsonRpcSubmitter(HttpClient httpClient, IAuth auth, string hostAddres _uri = new Uri(hostAddress); } - public async Task Submit(JsonRpc rpc) + public async Task Submit(JsonRpc rpc) { var request = new HttpRequestMessage(HttpMethod.Post, _uri) { @@ -32,7 +33,9 @@ public async Task Submit(JsonRpc rpc) var response = await _httpClient.SendAsync(request); if (response.StatusCode != HttpStatusCode.OK) { - throw new HttpRequestException($"Expected {HttpStatusCode.OK}, got {response.StatusCode}"); + return null; } + var content = await response.Content.ReadAsStreamAsync(); + return await JsonSerializer.DeserializeAsync(content); } } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs index bb9f5038829..5d2afe74df0 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/IJsonRpcSubmitter.cs @@ -1,9 +1,11 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Text.Json; + namespace Nethermind.Tools.Kute.JsonRpcSubmitter; interface IJsonRpcSubmitter { - Task Submit(JsonRpc rpc); + Task Submit(JsonRpc rpc); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs index d38cc4d410f..8c307bafc42 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Text.Json; using Nethermind.Tools.Kute.Auth; namespace Nethermind.Tools.Kute.JsonRpcSubmitter; @@ -14,9 +15,9 @@ public NullJsonRpcSubmitter(IAuth auth) _auth = auth; } - public Task Submit(JsonRpc rpc) + public Task Submit(JsonRpc rpc) { _ = _auth.AuthToken; - return Task.CompletedTask; + return Task.FromResult(null); } } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs new file mode 100644 index 00000000000..cb8233357bc --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public interface IJsonRpcValidator +{ + bool IsValid(JsonDocument? document); + bool IsInvalid(JsonDocument? document) => !IsValid(document); +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs new file mode 100644 index 00000000000..5c2c85c2a9a --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public class NonErrorJsonRpcValidator : IJsonRpcValidator +{ + + public bool IsValid(JsonDocument? document) + { + if (document is null) + { + return false; + } + + return !document.RootElement.TryGetProperty("error", out _); + } +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs new file mode 100644 index 00000000000..fab6856a813 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public class NullJsonRpcValidator : IJsonRpcValidator +{ + public bool IsValid(JsonDocument? document) => true; +} diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index d2697469f04..bd7b53bbe2c 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -6,6 +6,7 @@ using Nethermind.Tools.Kute.Auth; using Nethermind.Tools.Kute.JsonRpcMethodFilter; using Nethermind.Tools.Kute.JsonRpcSubmitter; +using Nethermind.Tools.Kute.JsonRpcValidator; using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; @@ -47,6 +48,11 @@ static IServiceProvider BuildServiceProvider(Config config) ); collection.AddSingleton>(new FileMessageProvider(config.MessagesFilePath)); collection.AddSingleton, JsonRpcMessageProvider>(); + collection.AddSingleton( + config.DryRun + ? new NullJsonRpcValidator() + : new NonErrorJsonRpcValidator() + ); collection.AddSingleton( new ComposedJsonRpcMethodFilter( config.MethodFilters @@ -80,7 +86,7 @@ static IServiceProvider BuildServiceProvider(Config config) return new NullProgressReporter(); }); collection.AddSingleton(); - collection.AddSingleton(_ => + collection.AddSingleton( config.MetricsOutputFormatter switch { MetricsOutputFormatter.Report => new MetricsTextOutputFormatter(), From 2d4801333195617033f08d5ca19b2997de45535a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 21 Jul 2023 17:28:58 -0300 Subject: [PATCH 04/23] Include more examples of filters and limits --- tools/Nethermind.Tools.Kute/README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/README.md b/tools/Nethermind.Tools.Kute/README.md index 80379a055d3..9870899e22b 100644 --- a/tools/Nethermind.Tools.Kute/README.md +++ b/tools/Nethermind.Tools.Kute/README.md @@ -34,16 +34,22 @@ Some typical usages are as follow: -i /rpc-logs -s keystore/jwt-secret -o Json ``` -### Use a single message file, using only `engine_` and `eth_` methods +### Use a single message file, using only `engine` and `eth` methods ``` --i /rpc.0 -s keystore/jwt-secret -f engine_, eth_ +-i /rpc.0 -s keystore/jwt-secret -f engine, eth ``` -### Use a single message file, using only the first 100 `engine_` methods +### Use a single message file, using only the first 100 methods ``` --i /rpc.0 -s keystore/jwt-secret -f engine_=100 +-i /rpc.0 -s keystore/jwt-secret -f .*=100 +``` + +### Use a single message file, using only the first 50 `engine_newPayloadV2` or `engine_newPayloadV3` methods + +``` +-i /rpc.0 -s keystore/jwt-secret -f engine_newPayloadV[23]=50 ``` ### Connect to a Nethermind Client running in a specific address and TTL From 6ac09f0f821515f7bd73ebf329e01cb94b032e0d Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Mon, 24 Jul 2023 10:15:38 -0300 Subject: [PATCH 05/23] Perform validation outside time measurement --- tools/Nethermind.Tools.Kute/Application.cs | 24 ++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index aa160af9c13..0d5e0cc8580 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Text.Json; using Nethermind.Tools.Kute.Extensions; using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.JsonRpcMethodFilter; @@ -54,19 +55,22 @@ public async Task Run() switch (jsonRpc) { case null: + { _metrics.TickFailed(); break; - + } case JsonRpc.BatchJsonRpc batch: + { using (_metrics.TimeBatch()) { await _submitter.Submit(batch); } break; - + } case JsonRpc.SingleJsonRpc single: + { if (single.IsResponse) { _metrics.TickResponses(); @@ -85,19 +89,23 @@ public async Task Run() continue; } + JsonDocument? result; using (_metrics.TimeMethod(single.MethodName)) { - var result = await _submitter.Submit(single); - if (_validator.IsInvalid(result)) - { - _metrics.TickFailed(); - } + result = await _submitter.Submit(single); } - break; + if (_validator.IsInvalid(result)) + { + _metrics.TickFailed(); + } + break; + } default: + { throw new ArgumentOutOfRangeException(nameof(jsonRpc)); + } } } } From a32b2bb0b10b0a709c0c1b81097a970efe9934c3 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 26 Jul 2023 16:39:55 -0300 Subject: [PATCH 06/23] Add 'TracerValidator' - Responses can be stored into a new file --- tools/Nethermind.Tools.Kute/Config.cs | 16 +++++++++-- .../JsonRpcValidator/TracerValidator.cs | 28 +++++++++++++++++++ tools/Nethermind.Tools.Kute/Program.cs | 19 +++++++++---- 3 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index 2890e9bd94d..c3d72d2d5dd 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -21,7 +21,7 @@ public class Config longName: "address", Required = false, Default = "http://localhost:8551", - HelpText = "Address where to send JSON RPC calls" + HelpText = "Address where to send JSON RPC requests" )] public string HostAddress { get; } @@ -79,6 +79,15 @@ public class Config )] public IEnumerable MethodFilters { get; } + [Option( + shortName: 't', + longName: "trace", + Required = false, + Default = null, + HelpText = "An optional file to store JSON-RPC responses" + )] + public string? ResponsesTraceFile { get; } + public Config( string messagesFilePath, string hostAddress, @@ -87,7 +96,8 @@ public Config( bool dryRun, bool showProgress, MetricsOutputFormatter metricsOutputFormatter, - IEnumerable methodFilters + IEnumerable methodFilters, + string? responsesTraceFile ) { MessagesFilePath = messagesFilePath; @@ -98,6 +108,6 @@ IEnumerable methodFilters ShowProgress = showProgress; MetricsOutputFormatter = metricsOutputFormatter; MethodFilters = methodFilters; - ShowProgress = showProgress; + ResponsesTraceFile = responsesTraceFile; } } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs new file mode 100644 index 00000000000..36b5028ca69 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public class TracerValidator : IJsonRpcValidator +{ + private readonly IJsonRpcValidator _validator; + private readonly string _tracesFilePath; + + public TracerValidator(IJsonRpcValidator validator, string tracesFilePath) + { + _validator = validator; + _tracesFilePath = tracesFilePath; + } + + public bool IsValid(JsonDocument? document) + { + using (StreamWriter sw = File.Exists(_tracesFilePath) ? File.CreateText(_tracesFilePath) : File.AppendText(_tracesFilePath)) + { + sw.WriteLine(document?.RootElement.ToString() ?? "null"); + } + + return _validator.IsValid(document); + } +} diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index bd7b53bbe2c..82bf13a380e 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -48,11 +48,20 @@ static IServiceProvider BuildServiceProvider(Config config) ); collection.AddSingleton>(new FileMessageProvider(config.MessagesFilePath)); collection.AddSingleton, JsonRpcMessageProvider>(); - collection.AddSingleton( - config.DryRun - ? new NullJsonRpcValidator() - : new NonErrorJsonRpcValidator() - ); + collection.AddSingleton(_ => + { + if (config.DryRun) + { + return new NullJsonRpcValidator(); + } + + var validator = new NonErrorJsonRpcValidator(); + if (config.ResponsesTraceFile is null) + { + return validator; + } + return new TracerValidator(validator, config.ResponsesTraceFile); + }); collection.AddSingleton( new ComposedJsonRpcMethodFilter( config.MethodFilters From 4af2f003b11c8efb2725f0c9f2a00a178253af40 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 26 Jul 2023 16:40:33 -0300 Subject: [PATCH 07/23] Document 'TracerValidator' - Disabled by default - Response tracing is ignored in dry mode --- tools/Nethermind.Tools.Kute/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/README.md b/tools/Nethermind.Tools.Kute/README.md index 9870899e22b..ade23e1ac2c 100644 --- a/tools/Nethermind.Tools.Kute/README.md +++ b/tools/Nethermind.Tools.Kute/README.md @@ -31,7 +31,13 @@ Some typical usages are as follow: ### Use a single messages file and emit results as JSON ``` --i /rpc-logs -s keystore/jwt-secret -o Json +-i /rpc.0 -s keystore/jwt-secret -o Json +``` + +### Use a single messages file and record all responses into a new file + +``` +-i /rpc.0 -s keystore/jwt-secret -t rpc.responses.txt ``` ### Use a single message file, using only `engine` and `eth` methods From 6f333c49e4dd4334f20ab88fb87ae6cfea9d210b Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 26 Jul 2023 16:47:36 -0300 Subject: [PATCH 08/23] Make config help text more consistent --- tools/Nethermind.Tools.Kute/Config.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index c3d72d2d5dd..adfa1800117 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -84,7 +84,7 @@ public class Config longName: "trace", Required = false, Default = null, - HelpText = "An optional file to store JSON-RPC responses" + HelpText = "Path to File to store JSON-RPC responses" )] public string? ResponsesTraceFile { get; } From b0919e6513bd90afba4181b02c6ef0cfa9bd5b05 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 26 Jul 2023 16:48:12 -0300 Subject: [PATCH 09/23] Capitalize --- tools/Nethermind.Tools.Kute/Config.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index adfa1800117..7251719e97d 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -29,7 +29,7 @@ public class Config shortName: 's', longName: "secret", Required = true, - HelpText = "Path to file with hex encoded secret for JWT authentication" + HelpText = "Path to File with hex encoded secret for JWT authentication" )] public string JwtSecretFilePath { get; } From 77ab7bd1abe9d762cb05c2168da53b5b81ba0cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Chodo=C5=82a?= <43241881+kamilchodola@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:50:41 +0200 Subject: [PATCH 10/23] Change name of tracing to verbose --- tools/Nethermind.Tools.Kute/Config.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index 7251719e97d..cdc3aa6e5ad 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -80,8 +80,8 @@ public class Config public IEnumerable MethodFilters { get; } [Option( - shortName: 't', - longName: "trace", + shortName: 'v', + longName: "verbose", Required = false, Default = null, HelpText = "Path to File to store JSON-RPC responses" From 5842d5c78e24b44c32b45a417ba0d69a14449280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Chodo=C5=82a?= <43241881+kamilchodola@users.noreply.github.com> Date: Thu, 27 Jul 2023 12:51:22 +0200 Subject: [PATCH 11/23] Update README.md --- tools/Nethermind.Tools.Kute/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/README.md b/tools/Nethermind.Tools.Kute/README.md index ade23e1ac2c..8742d4162a6 100644 --- a/tools/Nethermind.Tools.Kute/README.md +++ b/tools/Nethermind.Tools.Kute/README.md @@ -37,7 +37,7 @@ Some typical usages are as follow: ### Use a single messages file and record all responses into a new file ``` --i /rpc.0 -s keystore/jwt-secret -t rpc.responses.txt +-i /rpc.0 -s keystore/jwt-secret -v rpc.responses.txt ``` ### Use a single message file, using only `engine` and `eth` methods From 22d55df55c2c08d1ae16fe234773239a516ea7d1 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 27 Jul 2023 10:57:22 -0300 Subject: [PATCH 12/23] Use `-r|--responses` to store responses --- tools/Nethermind.Tools.Kute/Config.cs | 4 ++-- tools/Nethermind.Tools.Kute/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/Config.cs b/tools/Nethermind.Tools.Kute/Config.cs index 7251719e97d..b2bd10b2881 100644 --- a/tools/Nethermind.Tools.Kute/Config.cs +++ b/tools/Nethermind.Tools.Kute/Config.cs @@ -80,8 +80,8 @@ public class Config public IEnumerable MethodFilters { get; } [Option( - shortName: 't', - longName: "trace", + shortName: 'r', + longName: "responses", Required = false, Default = null, HelpText = "Path to File to store JSON-RPC responses" diff --git a/tools/Nethermind.Tools.Kute/README.md b/tools/Nethermind.Tools.Kute/README.md index ade23e1ac2c..891bbdaaac7 100644 --- a/tools/Nethermind.Tools.Kute/README.md +++ b/tools/Nethermind.Tools.Kute/README.md @@ -37,7 +37,7 @@ Some typical usages are as follow: ### Use a single messages file and record all responses into a new file ``` --i /rpc.0 -s keystore/jwt-secret -t rpc.responses.txt +-i /rpc.0 -s keystore/jwt-secret -r rpc.responses.txt ``` ### Use a single message file, using only `engine` and `eth` methods From 112836e2695c14d64bdcf64bc232e1b604054fb6 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 27 Jul 2023 11:12:18 -0300 Subject: [PATCH 13/23] Invert condition - Append if file exists - Create one if not --- tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs index 36b5028ca69..ccc07a9ac04 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs @@ -18,7 +18,7 @@ public TracerValidator(IJsonRpcValidator validator, string tracesFilePath) public bool IsValid(JsonDocument? document) { - using (StreamWriter sw = File.Exists(_tracesFilePath) ? File.CreateText(_tracesFilePath) : File.AppendText(_tracesFilePath)) + using (StreamWriter sw = File.Exists(_tracesFilePath) ? File.AppendText(_tracesFilePath) : File.CreateText(_tracesFilePath)) { sw.WriteLine(document?.RootElement.ToString() ?? "null"); } From 3d902c5906f7cb427c8566e9c4ea9564b33e8c88 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 27 Jul 2023 12:03:14 -0300 Subject: [PATCH 14/23] Make 'IJsonRpcValidator' take also request into consideration --- tools/Nethermind.Tools.Kute/Application.cs | 2 +- .../JsonRpcValidator/IJsonRpcValidator.cs | 4 ++-- .../JsonRpcValidator/NonErrorJsonRpcValidator.cs | 6 +++--- .../JsonRpcValidator/NullJsonRpcValidator.cs | 2 +- .../JsonRpcValidator/TracerValidator.cs | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index 0d5e0cc8580..a4036fefebd 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -95,7 +95,7 @@ public async Task Run() result = await _submitter.Submit(single); } - if (_validator.IsInvalid(result)) + if (_validator.IsInvalid(single, result)) { _metrics.TickFailed(); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs index cb8233357bc..20d30f714cc 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/IJsonRpcValidator.cs @@ -7,6 +7,6 @@ namespace Nethermind.Tools.Kute.JsonRpcValidator; public interface IJsonRpcValidator { - bool IsValid(JsonDocument? document); - bool IsInvalid(JsonDocument? document) => !IsValid(document); + bool IsValid(JsonRpc request, JsonDocument? response); + bool IsInvalid(JsonRpc request, JsonDocument? response) => !IsValid(request, response); } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs index 5c2c85c2a9a..fca69c7c04a 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs @@ -8,13 +8,13 @@ namespace Nethermind.Tools.Kute.JsonRpcValidator; public class NonErrorJsonRpcValidator : IJsonRpcValidator { - public bool IsValid(JsonDocument? document) + public bool IsValid(JsonRpc request, JsonDocument? response) { - if (document is null) + if (response is null) { return false; } - return !document.RootElement.TryGetProperty("error", out _); + return !response.RootElement.TryGetProperty("error", out _); } } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs index fab6856a813..12ab7833a45 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/NullJsonRpcValidator.cs @@ -7,5 +7,5 @@ namespace Nethermind.Tools.Kute.JsonRpcValidator; public class NullJsonRpcValidator : IJsonRpcValidator { - public bool IsValid(JsonDocument? document) => true; + public bool IsValid(JsonRpc request, JsonDocument? response) => true; } diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs index ccc07a9ac04..6fbaeab17bb 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs @@ -16,13 +16,13 @@ public TracerValidator(IJsonRpcValidator validator, string tracesFilePath) _tracesFilePath = tracesFilePath; } - public bool IsValid(JsonDocument? document) + public bool IsValid(JsonRpc request, JsonDocument? response) { using (StreamWriter sw = File.Exists(_tracesFilePath) ? File.AppendText(_tracesFilePath) : File.CreateText(_tracesFilePath)) { - sw.WriteLine(document?.RootElement.ToString() ?? "null"); + sw.WriteLine(response?.RootElement.ToString() ?? "null"); } - return _validator.IsValid(document); + return _validator.IsValid(request, response); } } From d0279fb205ddb5288fca9136f30eec2a93aedebf Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 27 Jul 2023 12:43:31 -0300 Subject: [PATCH 15/23] Validate 'NewPayload' responses using custom strategy - 'TracerValidator' is no longer a decorator --- .../ComposedJsonRpcValidator.cs | 18 ++++++++ .../Eth/NewPayloadJsonRpcValidator.cs | 46 +++++++++++++++++++ .../JsonRpcValidator/TracerValidator.cs | 15 +++--- tools/Nethermind.Tools.Kute/Program.cs | 14 ++++-- 4 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs create mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs new file mode 100644 index 00000000000..11d935cbc8a --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/ComposedJsonRpcValidator.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.JsonRpcValidator; + +public class ComposedJsonRpcValidator : IJsonRpcValidator +{ + private readonly IEnumerable _validators; + + public ComposedJsonRpcValidator(IEnumerable validators) + { + _validators = validators; + } + + public bool IsValid(JsonRpc request, JsonDocument? document) => _validators.All(validator => validator.IsValid(request, document)); +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs new file mode 100644 index 00000000000..dc302584065 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NewPayloadJsonRpcValidator.cs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace Nethermind.Tools.Kute.JsonRpcValidator.Eth; + +public class NewPayloadJsonRpcValidator : IJsonRpcValidator +{ + private readonly Regex _pattern = new Regex("engine_newPayload"); + + public bool IsValid(JsonRpc request, JsonDocument? response) + { + // If preconditions are not met, then mark it as Valid. + if (!ShouldValidateRequest(request) || response is null) + { + return true; + } + + if (!response.RootElement.TryGetProperty("result", out var result)) + { + return false; + } + + if (!result.TryGetProperty("status", out var status)) + { + return false; + } + + return status.GetString() == "VALID"; + } + + private bool ShouldValidateRequest(JsonRpc request) + { + if (request is JsonRpc.SingleJsonRpc { MethodName: not null } single) + { + if (_pattern.IsMatch(single.MethodName)) + { + return true; + } + } + + return false; + } +} diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs index 6fbaeab17bb..19a6d3cd0ff 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs @@ -7,22 +7,21 @@ namespace Nethermind.Tools.Kute.JsonRpcValidator; public class TracerValidator : IJsonRpcValidator { - private readonly IJsonRpcValidator _validator; private readonly string _tracesFilePath; - public TracerValidator(IJsonRpcValidator validator, string tracesFilePath) + public TracerValidator(string tracesFilePath) { - _validator = validator; _tracesFilePath = tracesFilePath; } public bool IsValid(JsonRpc request, JsonDocument? response) { - using (StreamWriter sw = File.Exists(_tracesFilePath) ? File.AppendText(_tracesFilePath) : File.CreateText(_tracesFilePath)) - { - sw.WriteLine(response?.RootElement.ToString() ?? "null"); - } + using StreamWriter sw = File.Exists(_tracesFilePath) + ? File.AppendText(_tracesFilePath) + : File.CreateText(_tracesFilePath); - return _validator.IsValid(request, response); + sw.WriteLine(response?.RootElement.ToString() ?? "null"); + + return true; } } diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index 82bf13a380e..671c76c3638 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -7,6 +7,7 @@ using Nethermind.Tools.Kute.JsonRpcMethodFilter; using Nethermind.Tools.Kute.JsonRpcSubmitter; using Nethermind.Tools.Kute.JsonRpcValidator; +using Nethermind.Tools.Kute.JsonRpcValidator.Eth; using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; @@ -55,12 +56,17 @@ static IServiceProvider BuildServiceProvider(Config config) return new NullJsonRpcValidator(); } - var validator = new NonErrorJsonRpcValidator(); - if (config.ResponsesTraceFile is null) + var validators = new List { - return validator; + new NonErrorJsonRpcValidator(), new NewPayloadJsonRpcValidator(), + }; + + if (config.ResponsesTraceFile is not null) + { + validators.Add(new TracerValidator(config.ResponsesTraceFile)); } - return new TracerValidator(validator, config.ResponsesTraceFile); + + return new ComposedJsonRpcValidator(validators); }); collection.AddSingleton( new ComposedJsonRpcMethodFilter( From 83fe482242a7ea10b985e7f9e8b02ea7dbcc721a Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 27 Jul 2023 12:56:55 -0300 Subject: [PATCH 16/23] Separate 'ResponseTracer' from Validator - Tracer should be independent - Add File and Null tracers - Simplify services setup --- tools/Nethermind.Tools.Kute/Application.cs | 6 ++++ .../{ => Eth}/NonErrorJsonRpcValidator.cs | 2 +- .../JsonRpcValidator/TracerValidator.cs | 27 --------------- tools/Nethermind.Tools.Kute/Program.cs | 33 ++++++++----------- .../ResponseTracer/FileResponseTracer.cs | 25 ++++++++++++++ .../ResponseTracer/IResponseTracer.cs | 11 +++++++ .../ResponseTracer/NullResponseTracer.cs | 11 +++++++ 7 files changed, 68 insertions(+), 47 deletions(-) rename tools/Nethermind.Tools.Kute/JsonRpcValidator/{ => Eth}/NonErrorJsonRpcValidator.cs (88%) delete mode 100644 tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs create mode 100644 tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs create mode 100644 tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs create mode 100644 tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index a4036fefebd..e9b1592ac4e 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -9,6 +9,7 @@ using Nethermind.Tools.Kute.JsonRpcValidator; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; +using Nethermind.Tools.Kute.ResponseTracer; namespace Nethermind.Tools.Kute; @@ -19,6 +20,7 @@ class Application private readonly IMessageProvider _msgProvider; private readonly IJsonRpcSubmitter _submitter; private readonly IJsonRpcValidator _validator; + private readonly IResponseTracer _responseTracer; private readonly IProgressReporter _progressReporter; private readonly IMetricsConsumer _metricsConsumer; private readonly IJsonRpcMethodFilter _methodFilter; @@ -27,6 +29,7 @@ public Application( IMessageProvider msgProvider, IJsonRpcSubmitter submitter, IJsonRpcValidator validator, + IResponseTracer responseTracer, IProgressReporter progressReporter, IMetricsConsumer metricsConsumer, IJsonRpcMethodFilter methodFilter @@ -35,6 +38,7 @@ IJsonRpcMethodFilter methodFilter _msgProvider = msgProvider; _submitter = submitter; _validator = validator; + _responseTracer = responseTracer; _progressReporter = progressReporter; _metricsConsumer = metricsConsumer; _methodFilter = methodFilter; @@ -100,6 +104,8 @@ public async Task Run() _metrics.TickFailed(); } + await _responseTracer.TraceResponse(result); + break; } default: diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs similarity index 88% rename from tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs rename to tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs index fca69c7c04a..3ac82ae65ed 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/NonErrorJsonRpcValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs @@ -3,7 +3,7 @@ using System.Text.Json; -namespace Nethermind.Tools.Kute.JsonRpcValidator; +namespace Nethermind.Tools.Kute.JsonRpcValidator.Eth; public class NonErrorJsonRpcValidator : IJsonRpcValidator { diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs deleted file mode 100644 index 19a6d3cd0ff..00000000000 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/TracerValidator.cs +++ /dev/null @@ -1,27 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited -// SPDX-License-Identifier: LGPL-3.0-only - -using System.Text.Json; - -namespace Nethermind.Tools.Kute.JsonRpcValidator; - -public class TracerValidator : IJsonRpcValidator -{ - private readonly string _tracesFilePath; - - public TracerValidator(string tracesFilePath) - { - _tracesFilePath = tracesFilePath; - } - - public bool IsValid(JsonRpc request, JsonDocument? response) - { - using StreamWriter sw = File.Exists(_tracesFilePath) - ? File.AppendText(_tracesFilePath) - : File.CreateText(_tracesFilePath); - - sw.WriteLine(response?.RootElement.ToString() ?? "null"); - - return true; - } -} diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index 671c76c3638..0e27abef2c2 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -11,6 +11,7 @@ using Nethermind.Tools.Kute.MessageProvider; using Nethermind.Tools.Kute.MetricsConsumer; using Nethermind.Tools.Kute.ProgressReporter; +using Nethermind.Tools.Kute.ResponseTracer; using Nethermind.Tools.Kute.SecretProvider; using Nethermind.Tools.Kute.SystemClock; @@ -49,25 +50,14 @@ static IServiceProvider BuildServiceProvider(Config config) ); collection.AddSingleton>(new FileMessageProvider(config.MessagesFilePath)); collection.AddSingleton, JsonRpcMessageProvider>(); - collection.AddSingleton(_ => - { - if (config.DryRun) - { - return new NullJsonRpcValidator(); - } - - var validators = new List - { - new NonErrorJsonRpcValidator(), new NewPayloadJsonRpcValidator(), - }; - - if (config.ResponsesTraceFile is not null) - { - validators.Add(new TracerValidator(config.ResponsesTraceFile)); - } - - return new ComposedJsonRpcValidator(validators); - }); + collection.AddSingleton( + config.DryRun + ? new NullJsonRpcValidator() + : new ComposedJsonRpcValidator(new List + { + new NonErrorJsonRpcValidator(), new NewPayloadJsonRpcValidator(), + }) + ); collection.AddSingleton( new ComposedJsonRpcMethodFilter( config.MethodFilters @@ -83,6 +73,11 @@ static IServiceProvider BuildServiceProvider(Config config) provider.GetRequiredService(), config.HostAddress )); + collection.AddSingleton( + config is { DryRun: false, ResponsesTraceFile: not null } + ? new FileResponseTracer(config.ResponsesTraceFile) + : new NullResponseTracer() + ); collection.AddSingleton(provider => { if (config.ShowProgress) diff --git a/tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs b/tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs new file mode 100644 index 00000000000..cd0a163ed29 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/ResponseTracer/FileResponseTracer.cs @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.ResponseTracer; + +public class FileResponseTracer : IResponseTracer +{ + private readonly string _tracesFilePath; + + public FileResponseTracer(string tracesFilePath) + { + _tracesFilePath = tracesFilePath; + } + + public async Task TraceResponse(JsonDocument? response) + { + await using StreamWriter sw = File.Exists(_tracesFilePath) + ? File.AppendText(_tracesFilePath) + : File.CreateText(_tracesFilePath); + + await sw.WriteLineAsync(response?.RootElement.ToString() ?? "null"); + } +} diff --git a/tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs b/tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs new file mode 100644 index 00000000000..f57bd7f68a8 --- /dev/null +++ b/tools/Nethermind.Tools.Kute/ResponseTracer/IResponseTracer.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.ResponseTracer; + +public interface IResponseTracer +{ + Task TraceResponse(JsonDocument? response); +} diff --git a/tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs b/tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs new file mode 100644 index 00000000000..6440e71612a --- /dev/null +++ b/tools/Nethermind.Tools.Kute/ResponseTracer/NullResponseTracer.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; + +namespace Nethermind.Tools.Kute.ResponseTracer; + +public class NullResponseTracer : IResponseTracer +{ + public Task TraceResponse(JsonDocument? response) => Task.CompletedTask; +} From 86d83b3edcaff0457c2a4b8e8df8a2bd04883d34 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Thu, 27 Jul 2023 12:58:47 -0300 Subject: [PATCH 17/23] Validate and trace batch responses --- tools/Nethermind.Tools.Kute/Application.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index e9b1592ac4e..9ee9a779533 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -66,11 +66,19 @@ public async Task Run() } case JsonRpc.BatchJsonRpc batch: { + JsonDocument? result; using (_metrics.TimeBatch()) { - await _submitter.Submit(batch); + result = await _submitter.Submit(batch); } + if (_validator.IsInvalid(batch, result)) + { + _metrics.TickFailed(); + } + + await _responseTracer.TraceResponse(result); + break; } case JsonRpc.SingleJsonRpc single: From c94c018c1707941bf9d36ae918c262aa03917e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Chodo=C5=82a?= Date: Thu, 27 Jul 2023 20:39:29 +0200 Subject: [PATCH 18/23] Add TickSucceeded --- tools/Nethermind.Tools.Kute/Application.cs | 4 ++++ tools/Nethermind.Tools.Kute/Metrics.cs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index 9ee9a779533..760cc1cccc5 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -111,6 +111,10 @@ public async Task Run() { _metrics.TickFailed(); } + else + { + _metrics.TickSucceeded(); + } await _responseTracer.TraceResponse(result); diff --git a/tools/Nethermind.Tools.Kute/Metrics.cs b/tools/Nethermind.Tools.Kute/Metrics.cs index f3038506d8b..2f384332554 100644 --- a/tools/Nethermind.Tools.Kute/Metrics.cs +++ b/tools/Nethermind.Tools.Kute/Metrics.cs @@ -23,6 +23,10 @@ public class Metrics { Name = "Failed", MeasurementUnit = Unit.Items, }; + private readonly CounterOptions _succeeded = new() + { + Name = "Succeeded", MeasurementUnit = Unit.Items, + }; private readonly CounterOptions _ignoredRequests = new() { Name = "Ignored Requests", MeasurementUnit = Unit.Items @@ -46,6 +50,7 @@ public Metrics() public void TickMessages() => _metrics.Measure.Counter.Increment(_messages); public void TickFailed() => _metrics.Measure.Counter.Increment(_failed); + public void TickSucceeded() => _metrics.Measure.Counter.Increment(_succeeded); public void TickIgnoredRequests() => _metrics.Measure.Counter.Increment(_ignoredRequests); public void TickResponses() => _metrics.Measure.Counter.Increment(_responses); From ec3bc7c77baf1e6d38d14a5f00b98ec1cea9986d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Chodo=C5=82a?= Date: Thu, 27 Jul 2023 22:01:23 +0200 Subject: [PATCH 19/23] Add temp sampleSize --- tools/Nethermind.Tools.Kute/Metrics.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/Metrics.cs b/tools/Nethermind.Tools.Kute/Metrics.cs index 2f384332554..5cafac756db 100644 --- a/tools/Nethermind.Tools.Kute/Metrics.cs +++ b/tools/Nethermind.Tools.Kute/Metrics.cs @@ -43,7 +43,8 @@ public class Metrics public Metrics() { - _metrics = new MetricsBuilder().Build(); + //TODO: HAAACK - this will use all requests from current set but thismagic number should be somehow replaced with all requests recognized at beggining + _metrics = new MetricsBuilder().SampleWith.AlgorithmR(10687).Build(); } public MetricsDataValueSource Snapshot => _metrics.Snapshot.Get(); From 96c46f045b80d34cefceabdf1ef5f5dd7ca0b200 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 28 Jul 2023 12:34:44 -0300 Subject: [PATCH 20/23] Tick succeeded on batches too --- tools/Nethermind.Tools.Kute/Application.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/Nethermind.Tools.Kute/Application.cs b/tools/Nethermind.Tools.Kute/Application.cs index 760cc1cccc5..9c6bff122fe 100644 --- a/tools/Nethermind.Tools.Kute/Application.cs +++ b/tools/Nethermind.Tools.Kute/Application.cs @@ -76,6 +76,10 @@ public async Task Run() { _metrics.TickFailed(); } + else + { + _metrics.TickSucceeded(); + } await _responseTracer.TraceResponse(result); From f9aca7829b7b451436c49a5e1f704708e301c016 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Fri, 28 Jul 2023 13:24:43 -0300 Subject: [PATCH 21/23] Use 'CompleteReservoir' for sampling - Removes need for hack/magic numbers --- .../Extensions/CompleteReservoir.cs | 45 +++++++++++++++++++ tools/Nethermind.Tools.Kute/Metrics.cs | 6 ++- 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs diff --git a/tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs b/tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs new file mode 100644 index 00000000000..b15ef8125bc --- /dev/null +++ b/tools/Nethermind.Tools.Kute/Extensions/CompleteReservoir.cs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using App.Metrics; +using App.Metrics.ReservoirSampling; +using App.Metrics.ReservoirSampling.Uniform; + +namespace Nethermind.Tools.Kute.Extensions; + +public class CompleteReservoir : IReservoir +{ + private const int DefaultSize = 10_000; + + private readonly List _values; + + public CompleteReservoir() : this(DefaultSize) { } + + public CompleteReservoir(int size) + { + _values = new List(size); + } + + public IReservoirSnapshot GetSnapshot(bool resetReservoir) + { + long count = _values.Count; + double sum = _values.Sum(v => v.Value); + IEnumerable values = _values.Select(v => v.Value); + + if (resetReservoir) + { + _values.Clear(); + } + + return new UniformSnapshot(count, sum, values); + } + + public IReservoirSnapshot GetSnapshot() => GetSnapshot(false); + + public void Reset() => _values.Clear(); + + public void Update(long value, string userValue) => _values.Add(new UserValueWrapper(value, userValue)); + + public void Update(long value) => _values.Add(new UserValueWrapper(value)); + +} diff --git a/tools/Nethermind.Tools.Kute/Metrics.cs b/tools/Nethermind.Tools.Kute/Metrics.cs index 5cafac756db..6c13a7e9cac 100644 --- a/tools/Nethermind.Tools.Kute/Metrics.cs +++ b/tools/Nethermind.Tools.Kute/Metrics.cs @@ -4,6 +4,7 @@ using App.Metrics; using App.Metrics.Counter; using App.Metrics.Timer; +using Nethermind.Tools.Kute.Extensions; namespace Nethermind.Tools.Kute; @@ -43,8 +44,9 @@ public class Metrics public Metrics() { - //TODO: HAAACK - this will use all requests from current set but thismagic number should be somehow replaced with all requests recognized at beggining - _metrics = new MetricsBuilder().SampleWith.AlgorithmR(10687).Build(); + _metrics = new MetricsBuilder() + .SampleWith.Reservoir() + .Build(); } public MetricsDataValueSource Snapshot => _metrics.Snapshot.Get(); From 6b8ac7be5b70b3d8d1cb0c2c2eea6951f89b0202 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 13 Dec 2023 10:19:24 -0300 Subject: [PATCH 22/23] Remove whitespace --- .../JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs index 3ac82ae65ed..e2af545fee1 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcValidator/Eth/NonErrorJsonRpcValidator.cs @@ -7,7 +7,6 @@ namespace Nethermind.Tools.Kute.JsonRpcValidator.Eth; public class NonErrorJsonRpcValidator : IJsonRpcValidator { - public bool IsValid(JsonRpc request, JsonDocument? response) { if (response is null) From 538d02e2517cabffb65da01204854ea596cb4509 Mon Sep 17 00:00:00 2001 From: Lautaro Emanuel Date: Wed, 13 Dec 2023 10:26:08 -0300 Subject: [PATCH 23/23] Remove `IAuth` from `NullJsonRpcSubmitter` --- .../JsonRpcSubmitter/NullJsonRpcSubmitter.cs | 13 +------------ tools/Nethermind.Tools.Kute/Program.cs | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs index 8c307bafc42..18e0ce01425 100644 --- a/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs +++ b/tools/Nethermind.Tools.Kute/JsonRpcSubmitter/NullJsonRpcSubmitter.cs @@ -2,22 +2,11 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Text.Json; -using Nethermind.Tools.Kute.Auth; namespace Nethermind.Tools.Kute.JsonRpcSubmitter; class NullJsonRpcSubmitter : IJsonRpcSubmitter { - private readonly IAuth _auth; - public NullJsonRpcSubmitter(IAuth auth) - { - _auth = auth; - } - - public Task Submit(JsonRpc rpc) - { - _ = _auth.AuthToken; - return Task.FromResult(null); - } + public Task Submit(JsonRpc rpc) => Task.FromResult(null); } diff --git a/tools/Nethermind.Tools.Kute/Program.cs b/tools/Nethermind.Tools.Kute/Program.cs index 0e27abef2c2..79932b07db9 100644 --- a/tools/Nethermind.Tools.Kute/Program.cs +++ b/tools/Nethermind.Tools.Kute/Program.cs @@ -66,13 +66,22 @@ static IServiceProvider BuildServiceProvider(Config config) ) ); collection.AddSingleton(provider => - config.DryRun - ? new NullJsonRpcSubmitter(provider.GetRequiredService()) - : new HttpJsonRpcSubmitter( + { + if (!config.DryRun) + { + return new HttpJsonRpcSubmitter( provider.GetRequiredService(), provider.GetRequiredService(), config.HostAddress - )); + ); + } + + // For dry runs we still want to trigger the generation of an AuthToken + // This is to ensure that all parameters required for the generation are correct, + // and not require a real run to verify that this is the case. + string _ = provider.GetRequiredService().AuthToken; + return new NullJsonRpcSubmitter(); + }); collection.AddSingleton( config is { DryRun: false, ResponsesTraceFile: not null } ? new FileResponseTracer(config.ResponsesTraceFile)