Exit early on error (#4220)

Exit early on error in steps except for step with "mustInitialize" set to false.
NethermindEth · Sep 27, 2022 · ef8cf02 · ef8cf02
1 parent 43fecc9
commit ef8cf02
Show file tree

Hide file tree

Showing 7 changed files with 104 additions and 60 deletions.
diff --git a/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs b/src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs
@@ -1,16 +1,16 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
@@ -135,40 +135,13 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken)
 
                 if (_logger.IsDebug) _logger.Debug($"Executing step: {stepInfo}");
 
-                Stopwatch stopwatch = Stopwatch.StartNew();
                 stepInfo.Stage = StepInitializationStage.Executing;
-                Task task = step.Execute(cancellationToken);
                 startedThisRound++;
-                Task continuationTask = task.ContinueWith(t =>
-                {
-                    stopwatch.Stop();
-
-                    if (t.IsFaulted && step.MustInitialize)
-                    {
-                        if (_logger.IsError) _logger.Error(
-                            $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms",
-                            t.Exception);
-                    }
-                    else if (t.IsFaulted)
-                    {
-                        if (_logger.IsWarn) _logger.Warn(
-                            $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms");
-                    }
-                    else
-                    {
-                        if (_logger.IsDebug) _logger.Debug(
-                            $"Step {step.GetType().Name.PadRight(24)} executed in {stopwatch.ElapsedMilliseconds}ms");
-                    }
-
-                    stepInfo.Stage = StepInitializationStage.Complete;
-                    _autoResetEvent.Set();
-
-                    if (_logger.IsDebug) _logger.Debug($"{step.GetType().Name.PadRight(24)} complete");
-                });
+                Task task = ExecuteStep(step, stepInfo, cancellationToken);
 
                 if (step.MustInitialize)
                 {
-                    _allPending.Enqueue(continuationTask);
+                    _allPending.Enqueue(task);
                 }
                 else
                 {
@@ -186,6 +159,46 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken)
             }
         }
 
+        private async Task ExecuteStep(IStep step, StepInfo stepInfo, CancellationToken cancellationToken)
+        {
+            Stopwatch stopwatch = Stopwatch.StartNew();
+            try
+            {
+                await step.Execute(cancellationToken);
+
+                if (_logger.IsDebug)
+                    _logger.Debug(
+                        $"Step {step.GetType().Name.PadRight(24)} executed in {stopwatch.ElapsedMilliseconds}ms");
+            }
+            catch (Exception exception)
+            {
+                if (step.MustInitialize)
+                {
+                    if (_logger.IsError)
+                        _logger.Error(
+                            $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms",
+                            exception);
+
+                    throw;
+                }
+
+                if (_logger.IsWarn)
+                {
+                    _logger.Warn(
+                        $"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms {exception}");
+                }
+            }
+            finally
+            {
+                stopwatch.Stop();
+
+                stepInfo.Stage = StepInitializationStage.Complete;
+                _autoResetEvent.Set();
+
+                if (_logger.IsDebug) _logger.Debug($"{step.GetType().Name.PadRight(24)} complete");
+            }
+        }
+
         private IStep? CreateStepInstance(StepInfo stepInfo)
         {
             IStep? step = null;

diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs
@@ -1,16 +1,16 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
@@ -43,5 +43,7 @@ public Task Execute(CancellationToken _)
 
             return Task.CompletedTask;
         }
+
+        public bool MustInitialize => false;
     }
 }
diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs b/src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs
@@ -53,6 +53,7 @@ public async Task Execute(CancellationToken cancellationToken)
                 catch (Exception e)
                 {
                     if (logger.IsError) logger.Error($"Failed to initialize plugin {plugin.Name} by {plugin.Author}", e);
+                    throw;
                 }
             }
         }

diff --git a/src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs b/src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs
@@ -1,16 +1,16 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
@@ -38,5 +38,7 @@ public Task Execute(CancellationToken cancellationToken)
             _api.Publishers.Add(logPublisher);
             return Task.CompletedTask;
         }
+
+        public bool MustInitialize => false;
     }
 }
diff --git a/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs b/src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs
@@ -1,19 +1,19 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
-// 
+//
 
 using System;
 using System.Collections.Generic;
@@ -74,4 +74,6 @@ await _api.MonitoringService.StartAsync().ContinueWith(x =>
                 logger.Info("Grafana / Prometheus metrics are disabled in configuration");
         }
     }
+
+    public bool MustInitialize => false;
 }
diff --git a/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs b/src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs
@@ -1,22 +1,23 @@
 //  Copyright (c) 2021 Demerzel Solutions Limited
 //  This file is part of the Nethermind library.
-// 
+//
 //  The Nethermind library is free software: you can redistribute it and/or modify
 //  it under the terms of the GNU Lesser General Public License as published by
 //  the Free Software Foundation, either version 3 of the License, or
 //  (at your option) any later version.
-// 
+//
 //  The Nethermind library is distributed in the hope that it will be useful,
 //  but WITHOUT ANY WARRANTY; without even the implied warranty of
 //  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 //  GNU Lesser General Public License for more details.
-// 
+//
 //  You should have received a copy of the GNU Lesser General Public License
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
 using System;
 using System.Threading;
 using System.Threading.Tasks;
+using FluentAssertions;
 using FluentAssertions.Execution;
 using Nethermind.Api;
 using Nethermind.Config;
@@ -26,6 +27,7 @@
 using Nethermind.Runner.Ethereum.Api;
 using Nethermind.Runner.Ethereum.Steps;
 using Nethermind.Serialization.Json;
+using NSubstitute.ExceptionExtensions;
 using NUnit.Framework;
 
 namespace Nethermind.Runner.Test.Ethereum.Steps
@@ -85,18 +87,15 @@ public async Task With_steps_from_here_AuRa()
                 runnerContext,
                 LimboLogs.Instance);
 
-            using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(1));
+            using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2));
 
             try
             {
                 await stepsManager.InitializeAll(source.Token);
             }
             catch (Exception e)
             {
-                if (!(e is OperationCanceledException))
-                {
-                    throw new AssertionFailedException($"Exception should be {nameof(OperationCanceledException)}");
-                }
+                e.Should().BeOfType<TestException>();
             }
         }
 
@@ -111,7 +110,7 @@ public async Task With_failing_steps()
                 runnerContext,
                 LimboLogs.Instance);
 
-            using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(1));
+            using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2));
 
             try
             {
@@ -211,7 +210,7 @@ public StepCAuRa(AuRaNethermindApi runnerContext)
 
         public override async Task Execute(CancellationToken cancellationToken)
         {
-            await Task.Run(() => throw new Exception());
+            await Task.Run(() => throw new TestException());
         }
     }
 
@@ -221,4 +220,8 @@ public StepCStandard(NethermindApi runnerContext)
         {
         }
     }
+
+    class TestException : Exception
+    {
+    }
 }
diff --git a/src/Nethermind/Nethermind.Runner/Program.cs b/src/Nethermind/Nethermind.Runner/Program.cs
@@ -147,7 +147,6 @@ private static void Run(string[] args)
 
             app.OnExecute(async () =>
             {
-                _appClosed.Reset();
                 IConfigProvider configProvider = BuildConfigProvider(app, loggerConfigSource, logLevelOverride, configsDirectory, configFile);
                 IInitConfig initConfig = configProvider.GetConfig<IInitConfig>();
                 IKeyStoreConfig keyStoreConfig = configProvider.GetConfig<IKeyStoreConfig>();
@@ -192,25 +191,47 @@ private static void Run(string[] args)
                 INethermindApi nethermindApi = apiBuilder.Create(plugins.OfType<IConsensusPlugin>());
                 ((List<INethermindPlugin>)nethermindApi.Plugins).AddRange(plugins);
 
+                _appClosed.Reset();
                 EthereumRunner ethereumRunner = new(nethermindApi);
-                await ethereumRunner.Start(_processCloseCancellationSource.Token).ContinueWith(x =>
+                bool runFailed = false;
+                try
                 {
-                    if (x.IsFaulted && _logger.IsError)
-                        _logger.Error("Error during ethereum runner start", x.Exception);
-                });
+                    await ethereumRunner.Start(_processCloseCancellationSource.Token);
 
-                _ = await Task.WhenAny(_cancelKeySource.Task, _processExit.Task);
+                    _ = await Task.WhenAny(_cancelKeySource.Task, _processExit.Task);
+                }
+                catch (Exception e)
+                {
+                    if (_logger.IsError) _logger.Error("Error during ethereum runner start", e);
+                    runFailed = true;
+                    _processCloseCancellationSource.Cancel();
+                }
 
                 _logger.Info("Closing, please wait until all functions are stopped properly...");
                 await ethereumRunner.StopAsync();
                 _logger.Info("All done, goodbye!");
                 _appClosed.Set();
 
+                if (runFailed)
+                {
+                    return -1;
+                }
                 return 0;
             });
 
-            _ = app.Execute(args);
-            _appClosed.Wait();
+            try
+            {
+                Environment.ExitCode = app.Execute(args);
+            }
+            catch (Exception)
+            {
+                Environment.ExitCode = -1;
+                throw;
+            }
+            finally
+            {
+                _appClosed.Wait();
+            }
         }
 
         private static IntPtr OnResolvingUnmanagedDll(Assembly _, string nativeLibraryName)