Hardened Security Context for Elasticsearch

pkg/controller/beat/common/stackmon/stackmon_test.go

@@ -76,6 +76,11 @@ func TestMetricBeat(t *testing.T) {
 				ReadOnly:  true,
 				MountPath: "/etc/metricbeat-config",
 			},
+			{
+				Name:      "metricbeat-data",
+				ReadOnly:  false,
+				MountPath: "/usr/share/metricbeat/data",
+			},
 			{
 				Name:      "shared-data",
 				ReadOnly:  false,
@@ -129,6 +134,10 @@ output:
 					Name:         "beat-metricbeat-config",
 					VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "beat-beat-monitoring-metricbeat-config", Optional: pointer.Bool(false)}},
 				},
+				{
+					Name:         "metricbeat-data",
+					VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},
+				},
 				{
 					Name:         "shared-data",
 					VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}},

pkg/controller/common/defaults/pod_template.go

@@ -343,6 +343,22 @@ func (b *PodTemplateBuilder) WithPodSecurityContext(securityContext corev1.PodSe
 	return b
 }
 
+// WithContainersSecurityContext sets Containers and InitContainers SecurityContext.
+// Must be called once all the Containers and InitContainers have been set.
+func (b *PodTemplateBuilder) WithContainersSecurityContext(securityContext corev1.SecurityContext) *PodTemplateBuilder {
+	for i := range b.PodTemplate.Spec.Containers {
+		if b.PodTemplate.Spec.Containers[i].SecurityContext == nil {
+			b.PodTemplate.Spec.Containers[i].SecurityContext = securityContext.DeepCopy()
+		}
+	}
+	for i := range b.PodTemplate.Spec.InitContainers {
+		if b.PodTemplate.Spec.InitContainers[i].SecurityContext == nil {
+			b.PodTemplate.Spec.InitContainers[i].SecurityContext = securityContext.DeepCopy()
+		}
+	}
+	return b
+}
+
 func (b *PodTemplateBuilder) WithAutomountServiceAccountToken() *PodTemplateBuilder {
 	if b.PodTemplate.Spec.AutomountServiceAccountToken == nil {
 		t := true

pkg/controller/common/keystore/initcontainer.go

@@ -32,6 +32,8 @@ type InitContainerParameters struct {
 	// SkipInitializedFlag when true do not use a flag to ensure the keystore is created only once. This should only be set
 	// to true if the keystore can be forcibly recreated.
 	SkipInitializedFlag bool
+	// SecurityContext is the security context applied to the keystore container.
+	SecurityContext *corev1.SecurityContext
 }
 
 // script is a small bash script to create an Elastic Stack keystore,
@@ -84,7 +86,7 @@ func initContainer(
 		return corev1.Container{}, err
 	}
 
-	return corev1.Container{
+	container := corev1.Container{
 		// Image will be inherited from pod template defaults
 		ImagePullPolicy: corev1.PullIfNotPresent,
 		Name:            InitContainerName,
@@ -97,5 +99,11 @@ func initContainer(
 			secureSettingsSecret.VolumeMount(),
 		},
 		Resources: parameters.Resources,
-	}, nil
+	}
+
+	if parameters.SecurityContext != nil {
+		container.SecurityContext = parameters.SecurityContext
+	}
+
+	return container, nil
 }

pkg/controller/common/stackmon/sidecar.go

@@ -47,12 +47,17 @@ func NewMetricBeatSidecar(
 		return BeatSidecar{}, err
 	}
 	image := container.ImageRepository(container.MetricbeatImage, version)
-	return NewBeatSidecar(ctx, client, "metricbeat", image, resource, monitoring.GetMetricsAssociation(resource), baseConfig, sourceCaVolume)
+
+	// EmptyDir volume so that MetricBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true
+	emptyDir := volume.NewEmptyDirVolume("metricbeat-data", "/usr/share/metricbeat/data")
+	return NewBeatSidecar(ctx, client, "metricbeat", image, resource, monitoring.GetMetricsAssociation(resource), baseConfig, sourceCaVolume, emptyDir)
 }
 
 func NewFileBeatSidecar(ctx context.Context, client k8s.Client, resource monitoring.HasMonitoring, version string, baseConfig string, additionalVolume volume.VolumeLike) (BeatSidecar, error) {
 	image := container.ImageRepository(container.FilebeatImage, version)
-	return NewBeatSidecar(ctx, client, "filebeat", image, resource, monitoring.GetLogsAssociation(resource), baseConfig, additionalVolume)
+	// EmptyDir volume so that FileBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true
+	emptyDir := volume.NewEmptyDirVolume("filebeat-data", "/usr/share/filebeat/data")
+	return NewBeatSidecar(ctx, client, "filebeat", image, resource, monitoring.GetLogsAssociation(resource), baseConfig, additionalVolume, emptyDir)
 }
 
 // BeatSidecar helps with building a beat sidecar container to monitor an Elastic Stack application. It focuses on
@@ -65,7 +70,7 @@ type BeatSidecar struct {
 }
 
 func NewBeatSidecar(ctx context.Context, client k8s.Client, beatName string, image string, resource monitoring.HasMonitoring,
-	associations []commonv1.Association, baseConfig string, additionalVolume volume.VolumeLike,
+	associations []commonv1.Association, baseConfig string, additionalVolumes ...volume.VolumeLike,
 ) (BeatSidecar, error) {
 	// build the beat config
 	config, err := newBeatConfig(ctx, client, beatName, resource, associations, baseConfig)
@@ -75,8 +80,10 @@ func NewBeatSidecar(ctx context.Context, client k8s.Client, beatName string, ima
 
 	// add additional volume (ex: CA volume of the monitored ES for Metricbeat)
 	volumes := config.volumes
-	if additionalVolume != nil {
-		volumes = append(volumes, additionalVolume)
+	for _, additionalVolume := range additionalVolumes {
+		if additionalVolume != nil {
+			volumes = append(volumes, additionalVolume)
+		}
 	}
 
 	// prepare the volume mounts for the beat container from all provided volumes

pkg/controller/elasticsearch/driver/driver.go

@@ -42,6 +42,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/observer"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/reconcile"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/remotecluster"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/securitycontext"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/services"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/stackmon"
@@ -322,14 +323,18 @@ func (d *defaultDriver) Reconcile(ctx context.Context) *reconciler.Results {
 		}
 	}
 
+	keystoreParams := initcontainer.KeystoreParams
+	keystoreSecurityContext := securitycontext.For(d.Version, true)
+	keystoreParams.SecurityContext = &keystoreSecurityContext
+
 	// setup a keystore with secure settings in an init container, if specified by the user
 	keystoreResources, err := keystore.ReconcileResources(
 		ctx,
 		d,
 		&d.ES,
 		esv1.ESNamer,
 		label.NewLabels(k8s.ExtractNamespacedName(&d.ES)),
-		initcontainer.KeystoreParams,
+		keystoreParams,
 	)
 	if err != nil {
 		return results.WithError(err)

pkg/controller/elasticsearch/initcontainer/prepare_fs.go

@@ -60,27 +60,27 @@ var (
 		Array: []LinkedFile{
 			{
 				Source: stringsutil.Concat(esvolume.XPackFileRealmVolumeMountPath, "/", filerealm.UsersFile),
-				Target: stringsutil.Concat(EsConfigSharedVolume.ContainerMountPath, "/", filerealm.UsersFile),
+				Target: stringsutil.Concat(EsConfigSharedVolume.InitContainerMountPath, "/", filerealm.UsersFile),
 			},
 			{
 				Source: stringsutil.Concat(esvolume.XPackFileRealmVolumeMountPath, "/", user.RolesFile),
-				Target: stringsutil.Concat(EsConfigSharedVolume.ContainerMountPath, "/", user.RolesFile),
+				Target: stringsutil.Concat(EsConfigSharedVolume.InitContainerMountPath, "/", user.RolesFile),
 			},
 			{
 				Source: stringsutil.Concat(esvolume.XPackFileRealmVolumeMountPath, "/", filerealm.UsersRolesFile),
-				Target: stringsutil.Concat(EsConfigSharedVolume.ContainerMountPath, "/", filerealm.UsersRolesFile),
+				Target: stringsutil.Concat(EsConfigSharedVolume.InitContainerMountPath, "/", filerealm.UsersRolesFile),
 			},
 			{
 				Source: stringsutil.Concat(settings.ConfigVolumeMountPath, "/", settings.ConfigFileName),
-				Target: stringsutil.Concat(EsConfigSharedVolume.ContainerMountPath, "/", settings.ConfigFileName),
+				Target: stringsutil.Concat(EsConfigSharedVolume.InitContainerMountPath, "/", settings.ConfigFileName),
 			},
 			{
 				Source: stringsutil.Concat(esvolume.UnicastHostsVolumeMountPath, "/", esvolume.UnicastHostsFile),
-				Target: stringsutil.Concat(EsConfigSharedVolume.ContainerMountPath, "/", esvolume.UnicastHostsFile),
+				Target: stringsutil.Concat(EsConfigSharedVolume.InitContainerMountPath, "/", esvolume.UnicastHostsFile),
 			},
 			{
 				Source: stringsutil.Concat(esvolume.XPackFileRealmVolumeMountPath, "/", esvolume.ServiceAccountsFile),
-				Target: stringsutil.Concat(EsConfigSharedVolume.ContainerMountPath, "/", esvolume.ServiceAccountsFile),
+				Target: stringsutil.Concat(EsConfigSharedVolume.InitContainerMountPath, "/", esvolume.ServiceAccountsFile),
 			},
 		},
 	}
@@ -109,7 +109,6 @@ func NewPrepareFSInitContainer(transportCertificatesVolume volume.SecretVolume,
 	certificatesVolumeMount := transportCertificatesVolume.VolumeMount()
 	certificatesVolumeMount.MountPath = initContainerTransportCertificatesVolumeMountPath
 
-	privileged := false
 	volumeMounts := append(
 		// we will also inherit all volume mounts from the main container later on in the pod template builder
 		PluginVolumes.InitContainerVolumeMounts(),
@@ -125,13 +124,10 @@ func NewPrepareFSInitContainer(transportCertificatesVolume volume.SecretVolume,
 	container := corev1.Container{
 		ImagePullPolicy: corev1.PullIfNotPresent,
 		Name:            PrepareFilesystemContainerName,
-		SecurityContext: &corev1.SecurityContext{
-			Privileged: &privileged,
-		},
-		Env:          defaults.PodDownwardEnvVars(),
-		Command:      []string{"bash", "-c", path.Join(esvolume.ScriptsVolumeMountPath, PrepareFsScriptConfigKey)},
-		VolumeMounts: volumeMounts,
-		Resources:    defaultResources,
+		Env:             defaults.PodDownwardEnvVars(),
+		Command:         []string{"bash", "-c", path.Join(esvolume.ScriptsVolumeMountPath, PrepareFsScriptConfigKey)},
+		VolumeMounts:    volumeMounts,
+		Resources:       defaultResources,
 	}
 
 	return container, nil

pkg/controller/elasticsearch/initcontainer/prepare_fs_script.go

@@ -95,20 +95,6 @@ var scriptTemplate = template.Must(template.New("").Parse(
 
 	echo "Starting init script"
 
-	######################
-	#  Config linking    #
-	######################
-
-	# Link individual files from their mount location into the config dir
-	# to a volume, to be used by the ES container
-	ln_start=$(date +%s)
-	{{range .LinkedFiles.Array}}
-		echo "Linking {{.Source}} to {{.Target}}"
-		ln -sf {{.Source}} {{.Target}}
-	{{end}}
-	echo "File linking duration: $(duration $ln_start) sec."
-
-
 	######################
 	#  Files persistence #
 	######################
@@ -127,6 +113,19 @@ var scriptTemplate = template.Must(template.New("").Parse(
 	{{end}}
 	echo "Files copy duration: $(duration $mv_start) sec."
 
+	######################
+	#  Config linking    #
+	######################
+
+	# Link individual files from their mount location into the config dir
+	# to a volume, to be used by the ES container
+	ln_start=$(date +%s)
+	{{range .LinkedFiles.Array}}
+		echo "Linking {{.Source}} to {{.Target}}"
+		ln -sf {{.Source}} {{.Target}}
+	{{end}}
+	echo "File linking duration: $(duration $ln_start) sec."
+
 	######################
 	#  Volumes chown     #
 	######################

pkg/controller/elasticsearch/nodespec/podspec.go

@@ -24,6 +24,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/initcontainer"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/label"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/network"
+	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/securitycontext"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/settings"
 	"github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/stackmon"
 	esvolume "github.com/elastic/cloud-on-k8s/v2/pkg/controller/elasticsearch/volume"
@@ -99,6 +100,17 @@ func BuildPodTemplateSpec(
 	}
 	annotations := buildAnnotations(es, cfg, keystoreResources, esScripts.ResourceVersion)
 
+	// Attempt to detect if the default data directory is mounted in a volume.
+	// If not, it could be a bug, a misconfiguration, or a custom storage configuration that requires the user to
+	// explicitly set ReadOnlyRootFilesystem to true.
+	enableReadOnlyRootFilesystem := false
+	for _, volumeMount := range volumeMounts {
+		if volumeMount.Name == esvolume.ElasticsearchDataVolumeName {
+			enableReadOnlyRootFilesystem = true
+			break
+		}
+	}
+
 	// build the podTemplate until we have the effective resources configured
 	builder = builder.
 		WithLabels(labels).
@@ -115,6 +127,8 @@ func BuildPodTemplateSpec(
 		WithInitContainers(initContainers...).
 		// inherit all env vars from main containers to allow Elasticsearch tools that read ES config to work in initContainers
 		WithInitContainerDefaults(builder.MainContainer().Env...).
+		// set a default security context for both the Containers and the InitContainers
+		WithContainersSecurityContext(securitycontext.For(ver, enableReadOnlyRootFilesystem)).
 		WithPreStopHook(*NewPreStopHook())
 
 	builder, err = stackmon.WithMonitoring(ctx, client, builder, es)

pkg/controller/elasticsearch/nodespec/podspec_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/stretchr/testify/require"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	ptr "k8s.io/utils/pointer"
 
 	commonv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/common/v1"
 	esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1"
@@ -261,6 +262,14 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 		initContainers[i].Env = initContainerEnv
 		initContainers[i].VolumeMounts = append(initContainers[i].VolumeMounts, volumeMounts...)
 		initContainers[i].Resources = DefaultResources
+		initContainers[i].SecurityContext = &corev1.SecurityContext{
+			Capabilities: &corev1.Capabilities{
+				Drop: []corev1.Capability{"ALL"},
+			},
+			Privileged:               ptr.Bool(false),
+			ReadOnlyRootFilesystem:   ptr.Bool(false),
+			AllowPrivilegeEscalation: ptr.Bool(false),
+		}
 	}
 
 	// remove the prepare-fs init-container from comparison, it has its own volume mount logic
@@ -304,10 +313,27 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 				Env:          initContainerEnv,
 				VolumeMounts: volumeMounts,
 				Resources:    DefaultResources, // inherited from main container
+				SecurityContext: &corev1.SecurityContext{
+					Capabilities: &corev1.Capabilities{
+						Drop: []corev1.Capability{"ALL"},
+					},
+					Privileged: ptr.Bool(false),
+					// ReadOnlyRootFilesystem is expected to be false in this test because there is no data volume.
+					ReadOnlyRootFilesystem:   ptr.Bool(false),
+					AllowPrivilegeEscalation: ptr.Bool(false),
+				},
 			}),
 			Containers: []corev1.Container{
 				{
 					Name: "additional-container",
+					SecurityContext: &corev1.SecurityContext{
+						Capabilities: &corev1.Capabilities{
+							Drop: []corev1.Capability{"ALL"},
+						},
+						Privileged:               ptr.Bool(false),
+						ReadOnlyRootFilesystem:   ptr.Bool(false),
+						AllowPrivilegeEscalation: ptr.Bool(false),
+					},
 				},
 				{
 					Name:  "elasticsearch",
@@ -325,6 +351,14 @@ func TestBuildPodTemplateSpec(t *testing.T) {
 					Lifecycle: &corev1.Lifecycle{
 						PreStop: NewPreStopHook(),
 					},
+					SecurityContext: &corev1.SecurityContext{
+						Capabilities: &corev1.Capabilities{
+							Drop: []corev1.Capability{"ALL"},
+						},
+						Privileged:               ptr.Bool(false),
+						ReadOnlyRootFilesystem:   ptr.Bool(false),
+						AllowPrivilegeEscalation: ptr.Bool(false),
+					},
 				},
 			},
 			TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,

pkg/controller/elasticsearch/nodespec/volumes.go

@@ -60,6 +60,10 @@ func buildVolumes(
 		esvolume.FileSettingsVolumeName,
 		esvolume.FileSettingsVolumeMountPath,
 	)
+	tmpVolume := volume.NewEmptyDirVolume(
+		esvolume.TempVolumeName,
+		esvolume.TempVolumeMountPath,
+	)
 	// append future volumes from PVCs (not resolved to a claim yet)
 	persistentVolumes := make([]corev1.Volume, 0, len(nodeSpec.VolumeClaimTemplates))
 	for _, claimTemplate := range nodeSpec.VolumeClaimTemplates {
@@ -89,6 +93,7 @@ func buildVolumes(
 			scriptsVolume.Volume(),
 			configVolume.Volume(),
 			downwardAPIVolume.Volume(),
+			tmpVolume.Volume(),
 		)...)
 	if keystoreResources != nil {
 		volumes = append(volumes, keystoreResources.Volume)
@@ -106,6 +111,7 @@ func buildVolumes(
 		scriptsVolume.VolumeMount(),
 		configVolume.VolumeMount(),
 		downwardAPIVolume.VolumeMount(),
+		tmpVolume.VolumeMount(),
 	)
 
 	// version gate for the file-based settings volume and volumeMounts