Skip to content

Commit

Permalink
Exit early on error (#4220)
Browse files Browse the repository at this point in the history 
Exit early on error in steps except for step with "mustInitialize" set to false.
  • Loading branch information
@asdacap @dceleda
asdacap authored and dceleda committed Sep 12, 2022
1 parent 993d48c commit cd53c87
Show file tree
Hide file tree
Showing 7 changed files with 116 additions and 73 deletions.
85 changes: 49 additions & 36 deletions src/Nethermind/Nethermind.Init/Steps/EthereumStepsManager.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// Copyright (c) 2021 Demerzel Solutions Limited
// This file is part of the Nethermind library.
//
//
// The Nethermind library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// The Nethermind library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
//
// You should have received a copy of the GNU Lesser General Public License
// along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.

Expand Down Expand Up @@ -63,11 +63,11 @@ private async Task ReviewDependencies(CancellationToken cancellationToken)
{
_logger.Debug($"{stepInfo} is {stepInfo.Stage}");
}

await _autoResetEvent.WaitOneAsync(cancellationToken);

if (_logger.IsDebug) _logger.Debug("Reviewing steps manager dependencies");

changedAnything = false;
foreach (StepInfo stepInfo in _allSteps)
{
Expand Down Expand Up @@ -120,7 +120,7 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken)
foreach (StepInfo stepInfo in _allSteps)
{
cancellationToken.ThrowIfCancellationRequested();

if (stepInfo.Stage != StepInitializationStage.WaitingForExecution)
{
continue;
Expand All @@ -135,40 +135,13 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken)

if (_logger.IsDebug) _logger.Debug($"Executing step: {stepInfo}");

Stopwatch stopwatch = Stopwatch.StartNew();
stepInfo.Stage = StepInitializationStage.Executing;
Task task = step.Execute(cancellationToken);
startedThisRound++;
Task continuationTask = task.ContinueWith(t =>
{
stopwatch.Stop();
if (t.IsFaulted && step.MustInitialize)
{
if (_logger.IsError) _logger.Error(
$"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms",
t.Exception);
}
else if(t.IsFaulted)
{
if (_logger.IsWarn) _logger.Warn(
$"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms");
}
else
{
if (_logger.IsDebug) _logger.Debug(
$"Step {step.GetType().Name.PadRight(24)} executed in {stopwatch.ElapsedMilliseconds}ms");
}
stepInfo.Stage = StepInitializationStage.Complete;
_autoResetEvent.Set();
if (_logger.IsDebug) _logger.Debug($"{step.GetType().Name.PadRight(24)} complete");
});
Task task = ExecuteStep(step, stepInfo, cancellationToken);

if (step.MustInitialize)
{
_allPending.Enqueue(continuationTask);
_allPending.Enqueue(task);
}
else
{
Expand All @@ -186,6 +159,46 @@ private void RunOneRoundOfInitialization(CancellationToken cancellationToken)
}
}

private async Task ExecuteStep(IStep step, StepInfo stepInfo, CancellationToken cancellationToken)
{
Stopwatch stopwatch = Stopwatch.StartNew();
try
{
await step.Execute(cancellationToken);

if (_logger.IsDebug)
_logger.Debug(
$"Step {step.GetType().Name.PadRight(24)} executed in {stopwatch.ElapsedMilliseconds}ms");
}
catch (Exception exception)
{
if (step.MustInitialize)
{
if (_logger.IsError)
_logger.Error(
$"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms",
exception);

throw;
}

if (_logger.IsWarn)
{
_logger.Warn(
$"Step {step.GetType().Name.PadRight(24)} failed after {stopwatch.ElapsedMilliseconds}ms {exception}");
}
}
finally
{
stopwatch.Stop();

stepInfo.Stage = StepInitializationStage.Complete;
_autoResetEvent.Set();

if (_logger.IsDebug) _logger.Debug($"{step.GetType().Name.PadRight(24)} complete");
}
}

private IStep? CreateStepInstance(StepInfo stepInfo)
{
IStep? step = null;
Expand Down
10 changes: 6 additions & 4 deletions src/Nethermind/Nethermind.Init/Steps/InitializeNodeStats.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// Copyright (c) 2021 Demerzel Solutions Limited
// This file is part of the Nethermind library.
//
//
// The Nethermind library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// The Nethermind library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
//
// You should have received a copy of the GNU Lesser General Public License
// along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.

Expand All @@ -35,13 +35,15 @@ public InitializeNodeStats(INethermindApi api)
public Task Execute(CancellationToken _)
{
INetworkConfig config = _api.Config<INetworkConfig>();

// create shared objects between discovery and peer manager
NodeStatsManager nodeStatsManager = new(_api.TimerFactory, _api.LogManager, config.MaxCandidatePeerCount);
_api.NodeStatsManager = nodeStatsManager;
_api.DisposeStack.Push(nodeStatsManager);

return Task.CompletedTask;
}

public bool MustInitialize => false;
}
}
1 change: 1 addition & 0 deletions src/Nethermind/Nethermind.Init/Steps/InitializePlugins.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public async Task Execute(CancellationToken cancellationToken)
catch (Exception e)
{
if(logger.IsError) logger.Error($"Failed to initialize plugin {plugin.Name} by {plugin.Author}", e);
throw;
}
}
}
Expand Down
8 changes: 5 additions & 3 deletions src/Nethermind/Nethermind.Init/Steps/StartLogProducer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
// Copyright (c) 2021 Demerzel Solutions Limited
// This file is part of the Nethermind library.
//
//
// The Nethermind library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// The Nethermind library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
//
// You should have received a copy of the GNU Lesser General Public License
// along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.

Expand Down Expand Up @@ -38,5 +38,7 @@ public Task Execute(CancellationToken cancellationToken)
_api.Publishers.Add(logPublisher);
return Task.CompletedTask;
}

public bool MustInitialize => false;
}
}
22 changes: 12 additions & 10 deletions src/Nethermind/Nethermind.Init/Steps/StartMonitoring.cs
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
// Copyright (c) 2021 Demerzel Solutions Limited
// This file is part of the Nethermind library.
//
//
// The Nethermind library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// The Nethermind library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
//
// You should have received a copy of the GNU Lesser General Public License
// along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
//
//

using System;
using System.Collections.Generic;
Expand Down Expand Up @@ -42,13 +42,13 @@ public async Task Execute(CancellationToken cancellationToken)
{
IMetricsConfig metricsConfig = _api.Config<IMetricsConfig>();
ILogger logger = _api.LogManager.GetClassLogger();

// hacky
if (!string.IsNullOrEmpty(metricsConfig.NodeName))
{
_api.LogManager.SetGlobalVariable("nodeName", metricsConfig.NodeName);
}

if (metricsConfig.Enabled)
{
Metrics.Version = VersionToMetrics.ConvertToNumber(ProductInfo.Version);
Expand All @@ -57,21 +57,23 @@ public async Task Execute(CancellationToken cancellationToken)
IEnumerable<Type> metrics = new TypeDiscovery().FindNethermindTypes(nameof(Metrics));
foreach (Type metric in metrics)
{
_api.MonitoringService.RegisterMetrics(metric);
_api.MonitoringService.RegisterMetrics(metric);
}

await _api.MonitoringService.StartAsync().ContinueWith(x =>
{
if (x.IsFaulted && logger.IsError)
if (x.IsFaulted && logger.IsError)
logger.Error("Error during starting a monitoring.", x.Exception);
}, cancellationToken);

_api.DisposeStack.Push(new Reactive.AnonymousDisposable(() => _api.MonitoringService.StopAsync())); // do not await
}
else
{
if (logger.IsInfo)
if (logger.IsInfo)
logger.Info("Grafana / Prometheus metrics are disabled in configuration");
}
}

public bool MustInitialize => false;
}
24 changes: 13 additions & 11 deletions src/Nethermind/Nethermind.Runner.Test/Ethereum/Steps/EthereumStepsManagerTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,23 @@
// Copyright (c) 2021 Demerzel Solutions Limited
// This file is part of the Nethermind library.
//
//
// The Nethermind library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
//
// The Nethermind library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
//
// You should have received a copy of the GNU Lesser General Public License
// along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.

using System;
using System.Threading;
using System.Threading.Tasks;
using FluentAssertions;
using FluentAssertions.Execution;
using Nethermind.Api;
using Nethermind.Config;
Expand All @@ -26,6 +27,7 @@
using Nethermind.Runner.Ethereum.Api;
using Nethermind.Runner.Ethereum.Steps;
using Nethermind.Serialization.Json;
using NSubstitute.ExceptionExtensions;
using NUnit.Framework;

namespace Nethermind.Runner.Test.Ethereum.Steps
Expand Down Expand Up @@ -85,18 +87,15 @@ public async Task With_steps_from_here_AuRa()
runnerContext,
LimboLogs.Instance);

using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(1));
using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2));

try
{
await stepsManager.InitializeAll(source.Token);
}
catch (Exception e)
{
if (!(e is OperationCanceledException))
{
throw new AssertionFailedException($"Exception should be {nameof(OperationCanceledException)}");
}
e.Should().BeOfType<TestException>();
}
}

Expand All @@ -111,7 +110,7 @@ public async Task With_failing_steps()
runnerContext,
LimboLogs.Instance);

using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(1));
using CancellationTokenSource source = new CancellationTokenSource(TimeSpan.FromSeconds(2));

try
{
Expand All @@ -125,7 +124,7 @@ public async Task With_failing_steps()
}
}
}

private static T CreateApi<T>() where T : INethermindApi, new() =>
new T()
{
Expand Down Expand Up @@ -211,7 +210,7 @@ public StepCAuRa(AuRaNethermindApi runnerContext)

public override async Task Execute(CancellationToken cancellationToken)
{
await Task.Run(() => throw new Exception());
await Task.Run(() => throw new TestException());
}
}

Expand All @@ -221,4 +220,7 @@ public StepCStandard(NethermindApi runnerContext)
{
}
}

class TestException: Exception {
}
}
Loading

0 comments on commit cd53c87

Please sign in to comment.