chore: log health check failures in master logs (#9291)

determined-ai · May 2, 2024 · f1aa92e · f1aa92e
1 parent 7496445
commit f1aa92e
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 0 deletions.
diff --git a/master/internal/core.go b/master/internal/core.go
@@ -208,6 +208,7 @@ func (m *Master) healthCheck(ctx context.Context) model.HealthCheck {
 	hc.Database = model.Healthy
 	_, err := db.Bun().NewSelect().Table("cluster_id").Exists(ctx)
 	if err != nil {
+		log.WithError(err).Error("database marked as unhealthy")
 		hc.Database = model.Unhealthy
 	}
 

diff --git a/master/internal/rm/dispatcherrm/dispatcher_resource_manager.go b/master/internal/rm/dispatcherrm/dispatcher_resource_manager.go
@@ -265,6 +265,7 @@ func (m *DispatcherResourceManager) HealthCheck() []model.ResourceManagerHealth
 	status := model.Healthy
 	_, err := m.apiClient.getVersion(context.TODO(), m.syslog.WithField("caller", "HealthCheck"))
 	if err != nil {
+		m.syslog.WithError(err).Error("dispatcher resource manager marked as unhealthy")
 		status = model.Unhealthy
 	}
 

diff --git a/master/internal/rm/kubernetesrm/kubernetes_resource_manager_intg_test.go b/master/internal/rm/kubernetesrm/kubernetes_resource_manager_intg_test.go
@@ -695,6 +695,7 @@ func TestHealthCheck(t *testing.T) {
 			podInterfaces: map[string]typedV1.PodInterface{
 				"namespace": mockPodInterface,
 			},
+			syslog: logrus.WithField("namespace", "test"),
 		},
 	}
 

diff --git a/master/internal/rm/kubernetesrm/pods.go b/master/internal/rm/kubernetesrm/pods.go
@@ -323,6 +323,7 @@ func (p *pods) HealthStatus() model.HealthStatus {
 	for _, podInterface := range p.podInterfaces {
 		_, err := podInterface.List(context.TODO(), metaV1.ListOptions{Limit: 1})
 		if err != nil {
+			p.syslog.WithError(err).Error("kubernetes resource manager marked as unhealthy")
 			return model.Unhealthy
 		}
 		return model.Healthy