Return 503 on healthz with error

shared/prometheus/service.go

@@ -61,6 +61,7 @@ func (s *Service) healthzHandler(w http.ResponseWriter, r *http.Request) {
 		Status bool   `json:"status"`
 		Err    string `json:"error"`
 	}
+	var hasError bool
 	var statuses []serviceStatus
 	for k, v := range s.svcRegistry.Statuses() {
 		s := serviceStatus{
@@ -70,11 +71,18 @@ func (s *Service) healthzHandler(w http.ResponseWriter, r *http.Request) {
 		if v != nil {
 			s.Status = false
 			s.Err = v.Error()
+			if s.Err != "" {
+				hasError = true
+			}
 		}
 		statuses = append(statuses, s)
 	}
 	response.Data = statuses
 
+	if hasError {
+		w.WriteHeader(http.StatusServiceUnavailable)
+	}
+
 	// Handle plain text content.
 	if contentType := negotiateContentType(r); contentType == contentTypePlainText {
 		var buf bytes.Buffer

shared/prometheus/service_test.go

@@ -185,5 +185,8 @@ func TestContentNegotiation(t *testing.T) {
 		if !strings.Contains(body, expectedJSON) {
 			t.Errorf("Unexpected data, want: %q got %q", expectedJSON, body)
 		}
+		if rr.Code < 500 {
+			t.Errorf("Expected a server error response code, but got %d", rr.Code)
+		}
 	})
 }