Merge branch 'test' into main

bcgov · Nov 8, 2023 · 52f5097 · 52f5097
2 parents 303c2be + 8060d20
commit 52f5097
Show file tree

Hide file tree

Showing 9 changed files with 118 additions and 20 deletions.
diff --git a/backend/webapi/Infrastructure/HealthChecks/BackgroundWorkerHealthCheck.cs b/backend/webapi/Infrastructure/HealthChecks/BackgroundWorkerHealthCheck.cs
@@ -0,0 +1,18 @@
+namespace Pidp.Infrastructure.HealthChecks;
+
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+public class BackgroundWorkerHealthCheck : IHealthCheck
+{
+    public bool IsRunning { get; set; }
+
+    public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
+    {
+        if (this.IsRunning)
+        {
+            return Task.FromResult(HealthCheckResult.Healthy("The background service is running."));
+        }
+
+        return Task.FromResult(HealthCheckResult.Unhealthy("The background service has stopped."));
+    }
+}
diff --git a/backend/webapi/Infrastructure/HealthChecks/HealthCheckTag.cs b/backend/webapi/Infrastructure/HealthChecks/HealthCheckTag.cs
@@ -0,0 +1,28 @@
+
+namespace Pidp.Infrastructure.HealthChecks;
+
+public class HealthCheckTag
+{
+    /// <summary>
+    /// Represents a "backgroundServices" check.
+    /// </summary>
+    public static readonly HealthCheckTag BackgroundServices = new("background-services");
+
+    /// <summary>
+    /// Represents a "liveness" check.
+    /// A service that fails a liveness check is considered to be unrecoverable and has to be restarted by the orchestrator.
+    /// </summary>
+    public static readonly HealthCheckTag Liveness = new("liveness");
+
+    /// <summary>
+    /// Represents a "readiness" check.
+    /// A service that fails a readiness check is considered to be unable to serve traffic temporarily.
+    /// </summary>
+    public static readonly HealthCheckTag Readiness = new("readiness");
+
+    public string Value { get; }
+
+    private HealthCheckTag(string value) => this.Value = value;
+
+    public static implicit operator string(HealthCheckTag type) => type.Value;
+}
diff --git a/backend/webapi/Infrastructure/Services/PlrStatusUpdateSchedulingService.cs b/backend/webapi/Infrastructure/Services/PlrStatusUpdateSchedulingService.cs
@@ -1,15 +1,22 @@
 namespace Pidp.Infrastructure.Services;
 
+using Pidp.Infrastructure.HealthChecks;
+
 public class PlrStatusUpdateSchedulingService : BackgroundService
 {
+    private readonly BackgroundWorkerHealthCheck healthCheck;
     private readonly ILogger<PlrStatusUpdateSchedulingService> logger;
     private readonly IServiceScopeFactory scopeFactory;
     private readonly PeriodicTimer timer = new(TimeSpan.FromSeconds(10));
 
-    public PlrStatusUpdateSchedulingService(IServiceScopeFactory scopeFactory, ILogger<PlrStatusUpdateSchedulingService> logger)
+    public PlrStatusUpdateSchedulingService(
+        IServiceScopeFactory scopeFactory,
+        ILogger<PlrStatusUpdateSchedulingService> logger,
+        BackgroundWorkerHealthCheck healthCheck)
     {
         this.logger = logger;
         this.scopeFactory = scopeFactory;
+        this.healthCheck = healthCheck;
     }
 
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
@@ -18,6 +25,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
         try
         {
+            this.healthCheck.IsRunning = true;
             while (await this.timer.WaitForNextTickAsync(stoppingToken)
                 && !stoppingToken.IsCancellationRequested)
             {
@@ -30,13 +38,18 @@ await scope.ServiceProvider.GetRequiredService<IPlrStatusUpdateService>()
         {
             this.logger.LogServiceHasStoppedUnexpectedly(e);
         }
+        finally
+        {
+            this.healthCheck.IsRunning = false;
+        }
     }
 
     public override async Task StopAsync(CancellationToken cancellationToken)
     {
         this.logger.LogServiceIsStopping();
 
         await base.StopAsync(cancellationToken);
+        this.healthCheck.IsRunning = false;
     }
 }
 

diff --git a/backend/webapi/Startup.cs b/backend/webapi/Startup.cs
@@ -1,8 +1,9 @@
 namespace Pidp;
 
 using FluentValidation.AspNetCore;
-using MassTransit;
+using HealthChecks.ApplicationStatus.DependencyInjection;
 using MicroElements.Swashbuckle.FluentValidation.AspNetCore;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
 using Microsoft.AspNetCore.Mvc.ApplicationModels;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.OpenApi.Models;
@@ -18,6 +19,7 @@ namespace Pidp;
 using Pidp.Features;
 using Pidp.Infrastructure;
 using Pidp.Infrastructure.Auth;
+using Pidp.Infrastructure.HealthChecks;
 using Pidp.Infrastructure.HttpClients;
 using Pidp.Infrastructure.Services;
 using Pidp.Infrastructure.Queue;
@@ -42,7 +44,8 @@ public void ConfigureServices(IServiceCollection services)
             .AddScoped<IEmailService, EmailService>()
             .AddScoped<IPidpAuthorizationService, PidpAuthorizationService>()
             .AddScoped<IPlrStatusUpdateService, PlrStatusUpdateService>()
-            .AddSingleton<IClock>(SystemClock.Instance);
+            .AddSingleton<IClock>(SystemClock.Instance)
+            .AddSingleton<BackgroundWorkerHealthCheck>();
 
         services.AddControllers(options => options.Conventions.Add(new RouteTokenTransformerConvention(new KabobCaseParameterTransformer())))
             .AddFluentValidation(options => options.RegisterValidatorsFromAssemblyContaining<Startup>())
@@ -60,7 +63,11 @@ public void ConfigureServices(IServiceCollection services)
             .AsImplementedInterfaces()
             .WithTransientLifetime());
 
-        services.AddHealthChecks();
+        services.AddHealthChecks()
+            .AddApplicationStatus(tags: new[] { HealthCheckTag.Liveness.Value })
+            .AddCheck<BackgroundWorkerHealthCheck>("PlrStatusUpdateSchedulingService", tags: new[] { HealthCheckTag.BackgroundServices.Value })
+            .AddNpgSql(config.ConnectionStrings.PidpDatabase, tags: new[] { HealthCheckTag.Readiness.Value })
+            .AddRabbitMQ(new Uri(config.RabbitMQ.HostAddress), tags: new[] { HealthCheckTag.Readiness.Value });
 
         services.AddSwaggerGen(options =>
         {
@@ -118,7 +125,18 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
         app.UseEndpoints(endpoints =>
         {
             endpoints.MapControllers();
-            endpoints.MapHealthChecks("/health/liveness").AllowAnonymous();
+            endpoints.MapHealthChecks("/health/background-services", new HealthCheckOptions
+            {
+                Predicate = registration => registration.Tags.Contains(HealthCheckTag.BackgroundServices)
+            }).AllowAnonymous();
+            endpoints.MapHealthChecks("/health/liveness", new HealthCheckOptions
+            {
+                Predicate = registration => registration.Tags.Contains(HealthCheckTag.Liveness)
+            }).AllowAnonymous();
+            endpoints.MapHealthChecks("/health/readiness", new HealthCheckOptions
+            {
+                Predicate = registration => registration.Tags.Contains(HealthCheckTag.Readiness)
+            }).AllowAnonymous();
         });
     }
 }
diff --git a/backend/webapi/pidp.csproj b/backend/webapi/pidp.csproj
@@ -7,7 +7,9 @@
     <UserSecretsId>5c2dc965-00b4-4531-9ff0-9b37193ead9b</UserSecretsId>
   </PropertyGroup>
   <ItemGroup>
+    <PackageReference Include="AspNetCore.HealthChecks.ApplicationStatus" Version="6.0.0" />
     <PackageReference Include="AspNetCore.HealthChecks.NpgSql" Version="6.0.2" />
+    <PackageReference Include="AspNetCore.HealthChecks.RabbitMQ" Version="6.0.2" />
     <PackageReference Include="AutoMapper" Version="11.0.1" />
     <PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="11.0.0" />
     <PackageReference Include="DomainResult" Version="2.0.0" />

diff --git a/charts/frontend/templates/deployment.yaml b/charts/frontend/templates/deployment.yaml
@@ -60,12 +60,21 @@ spec:
               protocol: TCP
           livenessProbe:
             httpGet:
-              path: /
+              path: /health/liveness
               port: http
+            timeoutSeconds: 1
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 5
+            initialDelaySeconds: 30 
           readinessProbe:
             httpGet:
               path: /
               port: http
+            timeoutSeconds: 1
+            periodSeconds: 10
+            successThreshold: 1
+            failureThreshold: 3              
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           volumeMounts:

diff --git a/charts/plr-intake/templates/deployment.yaml b/charts/plr-intake/templates/deployment.yaml
@@ -99,6 +99,7 @@ spec:
             periodSeconds: 10
             successThreshold: 1
             failureThreshold: 3
+            initialDelaySeconds: 30                       
           startupProbe:
             httpGet:
               path: /health/liveness
@@ -110,8 +111,12 @@ spec:
             failureThreshold: 30
           # readinessProbe:
           #   httpGet:
-          #     path: /
+          #     path: /health/liveness
           #     port: http
+          #   timeoutSeconds: 1
+          #   periodSeconds: 10
+          #   successThreshold: 1
+          #   failureThreshold: 3
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}

diff --git a/charts/webapi/templates/deployment.yaml b/charts/webapi/templates/deployment.yaml
@@ -68,7 +68,7 @@ spec:
                   name: rabbitmq-password
                   key: password
             - name: RabbitMQ__HostAddress
-              value: "rabbitmq://pidp:$(RabbitMQ__Password)@rabbitmq:5672/{{ $release }}"
+              value: "amqp://pidp:$(RabbitMQ__Password)@rabbitmq:5672/{{ $release }}"
             - name: DB_HOST
               valueFrom:
                 configMapKeyRef:
@@ -119,22 +119,27 @@ spec:
             timeoutSeconds: 1
             periodSeconds: 10
             successThreshold: 1
-            failureThreshold: 3
-            initialDelaySeconds: 8            
-          startupProbe:
+            failureThreshold: 5
+            initialDelaySeconds: 30           
+          # startupProbe:
+          #   httpGet:
+          #     path: /health/liveness
+          #     port: http
+          #     scheme: HTTP
+          #   timeoutSeconds: 1
+          #   periodSeconds: 10
+          #   successThreshold: 1
+          #   failureThreshold: 30
+          #   initialDelaySeconds: 5
+          readinessProbe:
             httpGet:
-              path: /health/liveness
+              path: /health/readiness
               port: http
-              scheme: HTTP
             timeoutSeconds: 1
             periodSeconds: 10
             successThreshold: 1
-            failureThreshold: 30
-            initialDelaySeconds: 5
-          readinessProbe:
-            httpGet:
-              path: /health/liveness
-              port: http
+            failureThreshold: 5
+
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       {{- with .Values.nodeSelector }}

diff --git a/charts/webapi/values.yaml b/charts/webapi/values.yaml
@@ -98,4 +98,4 @@ tolerations: []
 affinity: {}
 
 rabbitmq:
-  host: rabbitmq://pidp:password@rabbitmq:5672/
+  host: amqp://pidp:password@rabbitmq:5672/