From 0c8ded9b06011593def7f03bf70fea6f9f3cee7b Mon Sep 17 00:00:00 2001 From: Michael Morello Date: Tue, 17 Dec 2019 08:29:09 +0100 Subject: [PATCH] Fix readiness probe (#2272) * Mount annotations in the Pod * Redirect errors to stderr --- pkg/controller/common/volume/downward_api.go | 48 +++++++++++++++++++ .../elasticsearch/nodespec/readiness_probe.go | 40 ++++++++++++---- .../elasticsearch/nodespec/volumes.go | 7 ++- pkg/controller/elasticsearch/volume/names.go | 4 ++ 4 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 pkg/controller/common/volume/downward_api.go diff --git a/pkg/controller/common/volume/downward_api.go b/pkg/controller/common/volume/downward_api.go new file mode 100644 index 0000000000..a0f3dde373 --- /dev/null +++ b/pkg/controller/common/volume/downward_api.go @@ -0,0 +1,48 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package volume + +import ( + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" + corev1 "k8s.io/api/core/v1" +) + +var downwardAPIVolume = corev1.Volume{ + Name: volume.DownwardAPIVolumeName, + VolumeSource: corev1.VolumeSource{ + DownwardAPI: &corev1.DownwardAPIVolumeSource{ + Items: []corev1.DownwardAPIVolumeFile{ + { + Path: volume.LabelsFile, + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "metadata.labels", + }, + }, + }, + }, + }, +} + +var downwardAPIVolumeMount = corev1.VolumeMount{ + Name: volume.DownwardAPIVolumeName, + MountPath: volume.DownwardAPIMountPath, + ReadOnly: true, +} + +type DownwardAPI struct{} + +var _ VolumeLike = DownwardAPI{} + +func (DownwardAPI) Name() string { + return volume.DownwardAPIVolumeName +} + +func (DownwardAPI) Volume() corev1.Volume { + return downwardAPIVolume +} + +func (DownwardAPI) VolumeMount() corev1.VolumeMount { + return downwardAPIVolumeMount +} diff --git a/pkg/controller/elasticsearch/nodespec/readiness_probe.go b/pkg/controller/elasticsearch/nodespec/readiness_probe.go index 4c013f6d40..0323a9d5b1 100644 --- a/pkg/controller/elasticsearch/nodespec/readiness_probe.go +++ b/pkg/controller/elasticsearch/nodespec/readiness_probe.go @@ -7,6 +7,7 @@ package nodespec import ( "path" + "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/label" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" corev1 "k8s.io/api/core/v1" ) @@ -28,7 +29,25 @@ func NewReadinessProbe() *corev1.Probe { const ReadinessProbeScriptConfigKey = "readiness-probe-script.sh" const ReadinessProbeScript = `#!/usr/bin/env bash -# Consider a node to be healthy if it responds to a simple GET on "/_cat/nodes?local" + +# fail should be called as a last resort to help the user to understand why the probe failed +function fail { + timestamp=$(date --iso-8601=seconds) + echo "{\"timestamp\": \"${timestamp}\", \"message\": \"readiness probe failed\", "$1"}" | tee /proc/1/fd/2 + exit 1 +} + +labels="` + volume.DownwardAPIMountPath + "/" + volume.LabelsFile + `" + +if [[ ! -f "${labels}" ]]; then + fail "\"reason\": \"${labels} does not exist\"" +fi + +# get Elasticsearch version from the downward API +version=$(grep "` + label.VersionLabelName + `" ${labels} | cut -d '=' -f 2) +# remove quotes +version=$(echo "${version}" | tr -d '"') + READINESS_PROBE_TIMEOUT=${READINESS_PROBE_TIMEOUT:=3} # Check if PROBE_PASSWORD_PATH is set, otherwise fall back to its former name in 1.0.0.beta-1: PROBE_PASSWORD_FILE @@ -46,14 +65,19 @@ else BASIC_AUTH='' fi -# request Elasticsearch -ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/_cat/nodes?local" -status=$(curl -o /dev/null -w "%{http_code}" --max-time $READINESS_PROBE_TIMEOUT -XGET -s -k ${BASIC_AUTH} $ENDPOINT) +# request Elasticsearch on / +ENDPOINT="${READINESS_PROBE_PROTOCOL:-https}://127.0.0.1:9200/" +status=$(curl -o /dev/null -w "%{http_code}" --max-time ${READINESS_PROBE_TIMEOUT} -XGET -s -k ${BASIC_AUTH} $ENDPOINT) +curl_rc=$? + +if [[ ${curl_rc} -ne 0 ]]; then + fail "\"curl_rc\": \"${curl_rc}\"" +fi -# ready if status code 200 -if [[ $status == "200" ]]; then - exit 0 +# ready if status code 200, 503 is tolerable if ES version is 6.x +if [[ ${status} == "200" ]] || [[ ${status} == "503" && ${version:0:2} == "6." ]]; then + exit 0 else - exit 1 + fail " \"status\": \"${status}\", \"version\":\"${version}\" " fi ` diff --git a/pkg/controller/elasticsearch/nodespec/volumes.go b/pkg/controller/elasticsearch/nodespec/volumes.go index 24c26f8b3f..535344269b 100644 --- a/pkg/controller/elasticsearch/nodespec/volumes.go +++ b/pkg/controller/elasticsearch/nodespec/volumes.go @@ -5,8 +5,6 @@ package nodespec import ( - corev1 "k8s.io/api/core/v1" - esv1 "github.com/elastic/cloud-on-k8s/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/pkg/controller/common/keystore" @@ -15,8 +13,11 @@ import ( "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/settings" "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/user" esvolume "github.com/elastic/cloud-on-k8s/pkg/controller/elasticsearch/volume" + corev1 "k8s.io/api/core/v1" ) +var downwardAPIVolume = volume.DownwardAPI{} + func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keystore.Resources) ([]corev1.Volume, []corev1.VolumeMount) { configVolume := settings.ConfigSecretVolume(esv1.StatefulSet(esName, nodeSpec.Name)) @@ -70,6 +71,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst httpCertificatesVolume.Volume(), scriptsVolume.Volume(), configVolume.Volume(), + downwardAPIVolume.Volume(), )...) if keystoreResources != nil { volumes = append(volumes, keystoreResources.Volume) @@ -86,6 +88,7 @@ func buildVolumes(esName string, nodeSpec esv1.NodeSet, keystoreResources *keyst httpCertificatesVolume.VolumeMount(), scriptsVolume.VolumeMount(), configVolume.VolumeMount(), + downwardAPIVolume.VolumeMount(), ) return volumes, volumeMounts diff --git a/pkg/controller/elasticsearch/volume/names.go b/pkg/controller/elasticsearch/volume/names.go index b242689564..1e193f41f7 100644 --- a/pkg/controller/elasticsearch/volume/names.go +++ b/pkg/controller/elasticsearch/volume/names.go @@ -35,4 +35,8 @@ const ( ScriptsVolumeName = "elastic-internal-scripts" ScriptsVolumeMountPath = "/mnt/elastic-internal/scripts" + + DownwardAPIVolumeName = "downward-api" + DownwardAPIMountPath = "/mnt/elastic-internal/downward-api" + LabelsFile = "labels" )