diff --git a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs index 7d05f0461a5..1449548b5d7 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs @@ -1,19 +1,19 @@ // Copyright (c) 2021 Demerzel Solutions Limited // This file is part of the Nethermind library. -// +// // The Nethermind library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // The Nethermind library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. -// +// // You should have received a copy of the GNU Lesser General Public License // along with the Nethermind. If not, see . -// +// using System.Collections.Generic; using System.Threading; @@ -78,7 +78,7 @@ public void Setup() new ChainHeadInfoProvider(specProvider, _blockTree, stateProvider), new TxPoolConfig(), new TxValidator(specProvider.ChainId), - LimboLogs.Instance, + LimboLogs.Instance, transactionComparerProvider.GetDefaultComparer()); BlockhashProvider blockhashProvider = new (_blockTree, LimboLogs.Instance); VirtualMachine virtualMachine = new( @@ -91,7 +91,7 @@ public void Setup() storageProvider, virtualMachine, LimboLogs.Instance); - + BlockProcessor blockProcessor = new ( MainnetSpecProvider.Instance, Always.Valid, @@ -138,9 +138,9 @@ public void Test() resetEvent.Set(); } }; - + _blockchainProcessor.Start(); - + _blockTree.SuggestBlock(block0); _blockTree.SuggestBlock(block1); _blockTree.SuggestBlock(block2); @@ -149,7 +149,7 @@ public void Test() _blockTree.SuggestBlock(block2B); resetEvent.WaitOne(2000); - + _blockTree.Head.Should().Be(block2B); events.Should().HaveCount(6); events[4].Hash.Should().Be(block1B.Hash); diff --git a/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs b/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs index 03ba19c0776..e1f6e64f817 100644 --- a/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs +++ b/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs @@ -1,16 +1,16 @@ // Copyright (c) 2021 Demerzel Solutions Limited // This file is part of the Nethermind library. -// +// // The Nethermind library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // The Nethermind library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. -// +// // You should have received a copy of the GNU Lesser General Public License // along with the Nethermind. If not, see . @@ -63,11 +63,11 @@ private async Task ReviewDependencies(CancellationToken cancellationToken) { _logger.Debug($"{stepInfo} is {stepInfo.Stage}"); } - + await _autoResetEvent.WaitOneAsync(cancellationToken); - + if (_logger.IsDebug) _logger.Debug("Reviewing steps manager dependencies"); - + changedAnything = false; foreach (StepInfo stepInfo in _allSteps) { @@ -120,7 +120,7 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken) foreach (StepInfo stepInfo in _allSteps) { cancellationToken.ThrowIfCancellationRequested(); - + if (stepInfo.Stage != StepInitializationStage.WaitingForExecution) { continue; @@ -135,40 +135,13 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken) if (_logger.IsDebug) _logger.Debug($"Executing step: {stepInfo}"); - Stopwatch stopwatch = Stopwatch.StartNew(); stepInfo.Stage = StepInitializationStage.Executing; - Task task = step.Execute(cancellationToken); startedThisRound++; - Task continuationTask = task.ContinueWith(t => - { - stopwatch.Stop(); - - if (t.IsFaulted && step.MustInitialize) - { - if (_logger.IsError) _logger.Error( - $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms", - t.Exception); - } - else if(t.IsFaulted) - { - if (_logger.IsWarn) _logger.Warn( - $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms"); - } - else - { - if (_logger.IsDebug) _logger.Debug( - $"Step {step.GetType().Name.PadRight(24)} executed in {stopwatch.ElapsedMilliseconds}ms"); - } - - stepInfo.Stage = StepInitializationStage.Complete; - _autoResetEvent.Set(); - - if (_logger.IsDebug) _logger.Debug($"{step.GetType().Name.PadRight(24)} complete"); - }); + Task task = ExecuteStep(step, stepInfo, cancellationToken); if (step.MustInitialize) { - _allPending.Enqueue(continuationTask); + _allPending.Enqueue(task); } else { @@ -186,6 +159,46 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken) } } + private async Task ExecuteStep(IStep step, StepInfo stepInfo, CancellationToken cancellationToken) + { + Stopwatch stopwatch = Stopwatch.StartNew(); + try + { + await step.Execute(cancellationToken); + + if (_logger.IsDebug) + _logger.Debug( + $"Step {step.GetType().Name.PadRight(24)} executed in {stopwatch.ElapsedMilliseconds}ms"); + } + catch (Exception exception) + { + if (step.MustInitialize) + { + if (_logger.IsError) + _logger.Error( + $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms", + exception); + + throw; + } + + if (_logger.IsWarn) + { + _logger.Warn( + $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms {exception}"); + } + } + finally + { + stopwatch.Stop(); + + stepInfo.Stage = StepInitializationStage.Complete; + _autoResetEvent.Set(); + + if (_logger.IsDebug) _logger.Debug($"{step.GetType().Name.PadRight(24)} complete"); + } + } + private IStep? CreateStepInstance(StepInfo stepInfo) { IStep? step = null; diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs index 725b0f1d478..ebc1316470d 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs @@ -1,16 +1,16 @@ // Copyright (c) 2021 Demerzel Solutions Limited // This file is part of the Nethermind library. -// +// // The Nethermind library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // The Nethermind library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. -// +// // You should have received a copy of the GNU Lesser General Public License // along with the Nethermind. If not, see . @@ -35,7 +35,7 @@ public InitializeNodeStats(INethermindApi api) public Task Execute(CancellationToken _) { INetworkConfig config = _api.Config(); - + // create shared objects between discovery and peer manager NodeStatsManager nodeStatsManager = new(_api.TimerFactory, _api.LogManager, config.MaxCandidatePeerCount); _api.NodeStatsManager = nodeStatsManager; @@ -43,5 +43,7 @@ public Task Execute(CancellationToken _) return Task.CompletedTask; } + + public bool MustInitialize => false; } } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs b/src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs index 22d4bdcf022..0565bc560ba 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs @@ -53,6 +53,7 @@ public async Task Execute(CancellationToken cancellationToken) catch (Exception e) { if(logger.IsError) logger.Error($"Failed to initialize plugin {plugin.Name} by {plugin.Author}", e); + throw; } } } diff --git a/src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs b/src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs index df90e010f82..870f7811cc0 100644 --- a/src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs +++ b/src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs @@ -1,16 +1,16 @@ // Copyright (c) 2021 Demerzel Solutions Limited // This file is part of the Nethermind library. -// +// // The Nethermind library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // The Nethermind library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. -// +// // You should have received a copy of the GNU Lesser General Public License // along with the Nethermind. If not, see . @@ -38,5 +38,7 @@ public Task Execute(CancellationToken cancellationToken) _api.Publishers.Add(logPublisher); return Task.CompletedTask; } + + public bool MustInitialize => false; } } diff --git a/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs b/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs index fe199d4b33b..d92e65f8f0c 100644 --- a/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs +++ b/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs @@ -1,19 +1,19 @@ // Copyright (c) 2021 Demerzel Solutions Limited // This file is part of the Nethermind library. -// +// // The Nethermind library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // The Nethermind library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. -// +// // You should have received a copy of the GNU Lesser General Public License // along with the Nethermind. If not, see . -// +// using System; using System.Collections.Generic; @@ -42,13 +42,13 @@ public async Task Execute(CancellationToken cancellationToken) { IMetricsConfig metricsConfig = _api.Config(); ILogger logger = _api.LogManager.GetClassLogger(); - + // hacky if (!string.IsNullOrEmpty(metricsConfig.NodeName)) { _api.LogManager.SetGlobalVariable("nodeName", metricsConfig.NodeName); } - + if (metricsConfig.Enabled) { Metrics.Version = VersionToMetrics.ConvertToNumber(ProductInfo.Version); @@ -57,21 +57,23 @@ public async Task Execute(CancellationToken cancellationToken) IEnumerable metrics = new TypeDiscovery().FindNethermindTypes(nameof(Metrics)); foreach (Type metric in metrics) { - _api.MonitoringService.RegisterMetrics(metric); + _api.MonitoringService.RegisterMetrics(metric); } await _api.MonitoringService.StartAsync().ContinueWith(x => { - if (x.IsFaulted && logger.IsError) + if (x.IsFaulted && logger.IsError) logger.Error("Error during starting a monitoring.", x.Exception); }, cancellationToken); - + _api.DisposeStack.Push(new Reactive.AnonymousDisposable(() => _api.MonitoringService.StopAsync())); // do not await } else { - if (logger.IsInfo) + if (logger.IsInfo) logger.Info("Grafana / Prometheus metrics are disabled in configuration"); } } + + public bool MustInitialize => false; } diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs index 369a2c0f97f..06950aa626e 100644 --- a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs @@ -1,22 +1,23 @@ // Copyright (c) 2021 Demerzel Solutions Limited // This file is part of the Nethermind library. -// +// // The Nethermind library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. -// +// // The Nethermind library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. -// +// // You should have received a copy of the GNU Lesser General Public License // along with the Nethermind. If not, see . using System; using System.Threading; using System.Threading.Tasks; +using FluentAssertions; using FluentAssertions.Execution; using Nethermind.Api; using Nethermind.Config; @@ -26,6 +27,7 @@ using Nethermind.Runner.Ethereum.Api; using Nethermind.Runner.Ethereum.Steps; using Nethermind.Serialization.Json; +using NSubstitute.ExceptionExtensions; using NUnit.Framework; namespace Nethermind.Runner.Test.Ethereum.Steps @@ -85,7 +87,7 @@ public async Task With_steps_from_here_AuRa() runnerContext, LimboLogs.Instance); - using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2)); try { @@ -93,10 +95,7 @@ public async Task With_steps_from_here_AuRa() } catch (Exception e) { - if (!(e is OperationCanceledException)) - { - throw new AssertionFailedException($"Exception should be {nameof(OperationCanceledException)}"); - } + e.Should().BeOfType(); } } @@ -111,7 +110,7 @@ public async Task With_failing_steps() runnerContext, LimboLogs.Instance); - using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2)); try { @@ -125,7 +124,7 @@ public async Task With_failing_steps() } } } - + private static T CreateApi() where T : INethermindApi, new() => new T() { @@ -211,7 +210,7 @@ public StepCAuRa(AuRaNethermindApi runnerContext) public override async Task Execute(CancellationToken cancellationToken) { - await Task.Run(() => throw new Exception()); + await Task.Run(() => throw new TestException()); } } @@ -221,4 +220,7 @@ public StepCStandard(NethermindApi runnerContext) { } } + + class TestException: Exception { + } } diff --git a/src/Nethermind/Nethermind.Runner/Program.cs b/src/Nethermind/Nethermind.Runner/Program.cs index bccb658408e..ac9f852fe34 100644 --- a/src/Nethermind/Nethermind.Runner/Program.cs +++ b/src/Nethermind/Nethermind.Runner/Program.cs @@ -134,7 +134,6 @@ private static void Run(string[] args) app.OnExecute(async () => { - _appClosed.Reset(); IConfigProvider configProvider = BuildConfigProvider(app, loggerConfigSource, logLevelOverride, configsDirectory, configFile); IInitConfig initConfig = configProvider.GetConfig(); IKeyStoreConfig keyStoreConfig = configProvider.GetConfig(); @@ -178,25 +177,47 @@ private static void Run(string[] args) INethermindApi nethermindApi = apiBuilder.Create(plugins.OfType()); ((List)nethermindApi.Plugins).AddRange(plugins); + _appClosed.Reset(); EthereumRunner ethereumRunner = new(nethermindApi); - await ethereumRunner.Start(_processCloseCancellationSource.Token).ContinueWith(x => + bool runFailed = false; + try { - if (x.IsFaulted && _logger.IsError) - _logger.Error("Error during ethereum runner start", x.Exception); - }); - - _ = await Task.WhenAny(_cancelKeySource.Task, _processExit.Task); + await ethereumRunner.Start(_processCloseCancellationSource.Token); + + _ = await Task.WhenAny(_cancelKeySource.Task, _processExit.Task); + } + catch (Exception e) + { + if (_logger.IsError) _logger.Error("Error during ethereum runner start", e); + runFailed = true; + _processCloseCancellationSource.Cancel(); + } _logger.Info("Closing, please wait until all functions are stopped properly..."); await ethereumRunner.StopAsync(); _logger.Info("All done, goodbye!"); _appClosed.Set(); + if (runFailed) + { + return -1; + } return 0; }); - _ = app.Execute(args); - _appClosed.Wait(); + try + { + Environment.ExitCode = app.Execute(args); + } + catch (Exception) + { + Environment.ExitCode = -1; + throw; + } + finally + { + _appClosed.Wait(); + } } private static IntPtr OnResolvingUnmanagedDll(Assembly _, string nativeLibraryName)