Fix readiness probe (#2272)

* Mount annotations in the Pod

* Redirect errors to stderr

pkg/controller/common/volume/downward_api.go

@@ -0,0 +1,48 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package volume
+
+import (
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume"
+	corev1 "k8s.io/api/core/v1"
+)
+
+var downwardAPIVolume = corev1.Volume{
+	Name: volume.DownwardAPIVolumeName,
+	VolumeSource: corev1.VolumeSource{
+		DownwardAPI: &corev1.DownwardAPIVolumeSource{
+			Items: []corev1.DownwardAPIVolumeFile{
+				{
+					Path: volume.LabelsFile,
+					FieldRef: &corev1.ObjectFieldSelector{
+						FieldPath: "metadata.labels",
+					},
+				},
+			},
+		},
+	},
+}
+
+var downwardAPIVolumeMount = corev1.VolumeMount{
+	Name:      volume.DownwardAPIVolumeName,
+	MountPath: volume.DownwardAPIMountPath,
+	ReadOnly:  true,
+}
+
+type DownwardAPI struct{}
+
+var _ VolumeLike = DownwardAPI{}
+
+func (DownwardAPI) Name() string {
+	return volume.DownwardAPIVolumeName
+}
+
+func (DownwardAPI) Volume() corev1.Volume {
+	return downwardAPIVolume
+}
+
+func (DownwardAPI) VolumeMount() corev1.VolumeMount {
+	return downwardAPIVolumeMount
+}

pkg/controller/elasticsearch/nodespec/readiness_probe.go

@@ -7,6 +7,7 @@ package nodespec
 import (
 	"path"
 
+	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -28,7 +29,25 @@ func NewReadinessProbe() *corev1.Probe {
 
 const ReadinessProbeScriptConfigKey = "readiness-probe-script.sh"
 const ReadinessProbeScript = `#!/usr/bin/env bash
-# Consider a node to be healthy if it responds to a simple GET on "/_cat/nodes?local"
+
+# fail should be called as a last resort to help the user to understand why the probe failed
+function fail {
+  timestamp=$(date --iso-8601=seconds)
+  echo "{\"timestamp\": \"${timestamp}\", \"message\": \"readiness probe failed\", "$1"}" | tee /proc/1/fd/2
+  exit 1
+}
+
+labels="` + volume.DownwardAPIMountPath + "/" + volume.LabelsFile + `"
+
+if [[ ! -f "${labels}" ]]; then
+  fail "\"reason\": \"${labels} does not exist\""
+fi
+
+# get Elasticsearch version from the downward API
+version=$(grep "` + label.VersionLabelName + `" ${labels} | cut -d '=' -f 2)
+# remove quotes
+version=$(echo "${version}" | tr -d '"')
+
 READINESS_PROBE_TIMEOUT=${READINESS_PROBE_TIMEOUT:=3}
 
 # Check if PROBE_PASSWORD_PATH is set, otherwise fall back to its former name in 1.0.0.beta-1: PROBE_PASSWORD_FILE
@@ -46,14 +65,19 @@ else
   BASIC_AUTH=''
 fi
 
-# request Elasticsearch
-ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/_cat/nodes?local"
-status=$(curl -o /dev/null -w "%{http_code}" --max-time $READINESS_PROBE_TIMEOUT -XGET -s -k ${BASIC_AUTH} $ENDPOINT)
+# request Elasticsearch on /
+ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/"
+status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -s -k ${BASIC_AUTH} $ENDPOINT)
+curl_rc=$?
+
+if [[ ${curl_rc} -ne 0 ]]; then
+  fail "\"curl_rc\": \"${curl_rc}\""
+fi
 
-# ready if status code 200
-if [[ $status == "200" ]]; then
-	exit 0
+# ready if status code 200, 503 is tolerable if ES version is 6.x
+if [[ ${status} == "200" ]] || [[ ${status} == "503" && ${version:0:2} == "6." ]]; then
+  exit 0
 else
-	exit 1
+  fail " \"status\": \"${status}\", \"version\":\"${version}\" "
 fi
 `

pkg/controller/elasticsearch/nodespec/volumes.go

@@ -5,8 +5,6 @@
 package nodespec
 
 import (
-	corev1 "k8s.io/api/core/v1"
-
 	esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/keystore"
@@ -15,8 +13,11 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/settings"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user"
 	esvolume "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume"
+	corev1 "k8s.io/api/core/v1"
 )
 
+var downwardAPIVolume = volume.DownwardAPI{}
+
 func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keystore.Resources) ([]corev1.Volume, []corev1.VolumeMount) {
 
 	configVolume := settings.ConfigSecretVolume(esv1.StatefulSet(esName, nodeSpec.Name))
@@ -70,6 +71,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst
 			httpCertificatesVolume.Volume(),
 			scriptsVolume.Volume(),
 			configVolume.Volume(),
+			downwardAPIVolume.Volume(),
 		)...)
 	if keystoreResources != nil {
 		volumes = append(volumes, keystoreResources.Volume)
@@ -86,6 +88,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst
 		httpCertificatesVolume.VolumeMount(),
 		scriptsVolume.VolumeMount(),
 		configVolume.VolumeMount(),
+		downwardAPIVolume.VolumeMount(),
 	)
 
 	return volumes, volumeMounts

pkg/controller/elasticsearch/volume/names.go

@@ -35,4 +35,8 @@ const (
 
 	ScriptsVolumeName      = "elastic-internal-scripts"
 	ScriptsVolumeMountPath = "/mnt/elastic-internal/scripts"
+
+	DownwardAPIVolumeName = "downward-api"
+	DownwardAPIMountPath  = "/mnt/elastic-internal/downward-api"
+	LabelsFile            = "labels"
 )