Allow custom certificates on the transport layer

config/crds/v1/all-crds.yaml

@@ -5178,6 +5178,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a references to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml

@@ -9600,6 +9600,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a references to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

config/recipes/elastic-agent/fleet-kubernetes-integration.yaml

@@ -7,6 +7,10 @@ spec:
   count: 1
   elasticsearchRef:
     name: elasticsearch
+  http:
+    tls:
+      selfSignedCertificate:
+        disabled: true
   config:
     xpack.fleet.agents.elasticsearch.hosts: ["https://elasticsearch-es-http.default.svc:9200"]
     xpack.fleet.agents.fleet_server.hosts: ["https://fleet-server-agent-http.default.svc:8220"]

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -5214,6 +5214,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a references to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

docs/orchestrating-elastic-stack-applications/elasticsearch/remote-clusters.asciidoc

@@ -95,23 +95,13 @@ kind: Elasticsearch
 metadata:
   name: cluster-two
 spec:
+  transport:
+    tls:
+      certificateAuthorities:
+        configMapName: remote-certs
   nodeSets:
-  - config:
-      xpack.security.transport.ssl.certificate_authorities:
-      - /usr/share/elasticsearch/config/other/remote.ca.crt
-    count: 3
+  - count: 3
     name: default
-    podTemplate:
-      spec:
-        containers:
-        - name: elasticsearch
-          volumeMounts:
-          - mountPath: /usr/share/elasticsearch/config/other
-            name: remote-certs
-        volumes:
-        - name: remote-certs
-          secret:
-            secretName: remote-certs
   version: {version}
 ----
 

docs/orchestrating-elastic-stack-applications/elasticsearch/reserved-settings.asciidoc

@@ -24,9 +24,7 @@ The following Elasticsearch settings are managed by ECK:
 * `xpack.security.http.ssl.certificate`
 * `xpack.security.http.ssl.enabled`
 * `xpack.security.http.ssl.key`
-* `xpack.security.transport.ssl.certificate`
 * `xpack.security.transport.ssl.enabled`
-* `xpack.security.transport.ssl.key`
 * `xpack.security.transport.ssl.verification_mode`
 
 The following Elasticsearch settings are not supported by ECK:

docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc

@@ -68,3 +68,93 @@ spec:
   - name: default
     count: 3
 ----
+
+== Issue node transport certificates with third-party tools
+
+When following the instructions in <<{p}-transport-ca>> the issuance of certificates is orchestrated by the ECK operator and the operator needs access to the CAs private key.
+If this is undesirable it is also possible to configure node transport certificates without involving the ECK operator. The following two pre-requisites apply:
+
+1. the tooling used must be able to issue individual certificates for each Elasticsearch node and dynamically add or remove certificates as the cluster scales up and down.
+2. the ECK operator must be configured to trust the CA in use.
+
+The following example configuration using link:https://cert-manager.io/docs/projects/csi-driver/[cert-manager csi-driver] and link:https://cert-manager.io/docs/projects/trust-manager/[trust-manager] meets these two requirements:
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: elasticsearch.k8s.elastic.co/{eck_crd_version}
+kind: Elasticsearch
+metadata:
+  name: es
+spec:
+  version: {version}
+  transport:
+    tls:
+      certificateAuthorities:
+        configMapName: trust
+  nodeSets:
+  - name: default
+    count: 3
+    config:
+      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/cert-manager-certs/tls.key
+      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/cert-manager-certs/tls.crt
+    podTemplate:
+      spec:
+        containers:
+        - name: elasticsearch
+          volumeMounts:
+          - name: transport-certs
+            mountPath: /usr/share/elasticsearch/config/cert-manager-certs
+        volumes:
+        - name: transport-certs
+          csi:
+            driver: csi.cert-manager.io
+            readOnly: true
+            volumeAttributes:
+              csi.cert-manager.io/issuer-name: ca-cluster-issuer
+              csi.cert-manager.io/issuer-kind: ClusterIssuer
+              csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local"
+----
+
+The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and that a trust bundle containing the PEM encoded CA certificate is available in a file called `ca.crt` in a ConfigMap called `trust` in the same namespace as the Elasticsearch resource. The following section is only provided to illustrate how these certificates can be configured in principle:
+
+[source,yaml]
+----
+apiVersion: trust.cert-manager.io/v1alpha1
+kind: Bundle
+metadata:
+  name: trust
+spec:
+  sources:
+  - secret:
+      name: "root-ca-secret"
+      key: "tls.crt"
+  target:
+    configMap:
+      key: "ca.crt"
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: selfsigned-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: selfsigned-ca
+  secretName: root-ca-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: ca-cluster-issuer
+spec:
+  ca:
+    secretName: root-ca-secret
+...
+----

docs/reference/api-docs.asciidoc

@@ -454,6 +454,23 @@ Config represents untyped YAML configuration.
 
 
 
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref"]
+=== ConfigMapRef 
+
+ConfigMapRef is a reference to a config map that exists in the same namespace as the referring resource.
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transporttlsoptions[$$TransportTLSOptions$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`configMapName`* __string__ | 
+|===
+
+
 [id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configsource"]
 === ConfigSource 
 
@@ -1383,6 +1400,7 @@ TransportConfig holds the transport layer settings for Elasticsearch.
 | *`subjectAltNames`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-subjectalternativename[$$SubjectAlternativeName$$] array__ | SubjectAlternativeNames is a list of SANs to include in the generated node transport TLS certificates.
 | *`certificate`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-secretref[$$SecretRef$$]__ | Certificate is a reference to a Kubernetes secret that contains the CA certificate and private key for generating node certificates. The referenced secret should contain the following: 
  - `ca.crt`: The CA certificate in PEM format. - `ca.key`: The private key for the CA certificate in PEM format.
+| *`certificateAuthorities`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref[$$ConfigMapRef$$]__ | CertificateAuthorities is a references to a config map that contains one or more x509 certificates for trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
 |===
 
 

pkg/apis/common/v1/common.go

@@ -45,6 +45,15 @@ func (ds DeploymentStatus) IsDegraded(prev DeploymentStatus) bool {
 	return prev.Health == GreenHealth && ds.Health != GreenHealth
 }
 
+// ConfigMapRef is a reference to a config map that exists in the same namespace as the referring resource.
+type ConfigMapRef struct {
+	ConfigMapName string `json:"configMapName,omitempty"`
+}
+
+func (c ConfigMapRef) IsDefined() bool {
+	return len(c.ConfigMapName) > 0
+}
+
 // SecretRef is a reference to a secret that exists in the same namespace.
 type SecretRef struct {
 	// SecretName is the name of the secret.

pkg/apis/elasticsearch/v1/elasticsearch_types.go

@@ -172,6 +172,9 @@ type TransportTLSOptions struct {
 	// - `ca.crt`: The CA certificate in PEM format.
 	// - `ca.key`: The private key for the CA certificate in PEM format.
 	Certificate commonv1.SecretRef `json:"certificate,omitempty"`
+	// CertificateAuthorities is a references to a config map that contains one or more x509 certificates for
+	// trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
+	CertificateAuthorities commonv1.ConfigMapRef `json:"certificateAuthorities,omitempty"`
 }
 
 func (tto TransportTLSOptions) UserDefinedCA() bool {

pkg/apis/elasticsearch/v1/fields.go

@@ -65,8 +65,6 @@ var UnsupportedSettings = []string{
 	XPackSecurityHttpSslCertificate,
 	XPackSecurityHttpSslEnabled,
 	XPackSecurityHttpSslKey,
-	XPackSecurityTransportSslCertificate,
 	XPackSecurityTransportSslEnabled,
-	XPackSecurityTransportSslKey,
 	XPackSecurityTransportSslVerificationMode,
 }

pkg/controller/elasticsearch/certificates/reconcile.go

@@ -100,6 +100,14 @@ func ReconcileTransport(
 	// label certificates secrets with the cluster name
 	certsLabels := label.NewLabels(k8s.ExtractNamespacedName(&es))
 
+	// Validate user defined additional trusted CAs config map exists and contains the right things.
+	// It will be mounted directly to the Pods and watched by Elasticsearch so not further reconciliation is needed.
+	additionalCAs, err := transport.MaybeRetrieveAdditionalCAs(ctx, driver.K8sClient(), es)
+	if err != nil {
+		driver.Recorder().Eventf(&es, corev1.EventTypeWarning, events.EventReasonUnexpected, err.Error())
+		return results.WithError(err)
+	}
+
 	// reconcile transport CA and certs
 	transportCA, err := transport.ReconcileOrRetrieveCA(
 		ctx,
@@ -120,7 +128,7 @@ func ReconcileTransport(
 	)
 
 	// reconcile transport public certs secret
-	if err := transport.ReconcileTransportCertsPublicSecret(ctx, driver.K8sClient(), es, transportCA); err != nil {
+	if err := transport.ReconcileTransportCertsPublicSecret(ctx, driver.K8sClient(), es, transportCA, additionalCAs); err != nil {
 		return results.WithError(err)
 	}
 
@@ -129,6 +137,7 @@ func ReconcileTransport(
 		ctx,
 		driver.K8sClient(),
 		transportCA,
+		additionalCAs,
 		es,
 		certRotation,
 	)

pkg/controller/elasticsearch/certificates/transport/public_secret.go

@@ -5,6 +5,7 @@
 package transport
 
 import (
+	"bytes"
 	"context"
 
 	corev1 "k8s.io/api/core/v1"
@@ -24,6 +25,7 @@ func ReconcileTransportCertsPublicSecret(
 	c k8s.Client,
 	es esv1.Elasticsearch,
 	ca *certificates.CA,
+	additionalCAs []byte,
 ) error {
 	esNSN := k8s.ExtractNamespacedName(&es)
 	meta := k8s.ToObjectMeta(PublicCertsSecretRef(esNSN))
@@ -32,7 +34,7 @@ func ReconcileTransportCertsPublicSecret(
 	expected := corev1.Secret{
 		ObjectMeta: meta,
 		Data: map[string][]byte{
-			certificates.CAFileName: certificates.EncodePEMCert(ca.Cert.Raw),
+			certificates.CAFileName: bytes.Join([][]byte{certificates.EncodePEMCert(ca.Cert.Raw), additionalCAs}, nil),
 		},
 	}
 

pkg/controller/elasticsearch/certificates/transport/public_secret_test.go

@@ -5,6 +5,7 @@
 package transport
 
 import (
+	"bytes"
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
@@ -64,6 +65,7 @@ func TestReconcileTransportCertsPublicSecret(t *testing.T) {
 	tests := []struct {
 		name       string
 		client     func(*testing.T, ...runtime.Object) k8s.Client
+		extraCA    []byte
 		wantSecret func(*testing.T) *corev1.Secret
 		wantErr    bool
 	}{
@@ -82,6 +84,18 @@ func TestReconcileTransportCertsPublicSecret(t *testing.T) {
 			},
 			wantSecret: mkWantedSecret,
 		},
+		{
+			name:    "concatenates multiple CAs",
+			client:  mkClient,
+			extraCA: extraCA,
+			wantSecret: func(t *testing.T) *corev1.Secret {
+				t.Helper()
+				s := mkWantedSecret(t)
+				s.Data[certificates.CAFileName] = bytes.Join([][]byte{s.Data[certificates.CAFileName], extraCA}, nil)
+				return s
+			},
+			wantErr: false,
+		},
 		{
 			name: "removes extraneous keys",
 			client: func(t *testing.T, _ ...runtime.Object) k8s.Client {
@@ -131,7 +145,7 @@ func TestReconcileTransportCertsPublicSecret(t *testing.T) {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
 			client := tt.client(t)
-			err := ReconcileTransportCertsPublicSecret(context.Background(), client, *owner, ca)
+			err := ReconcileTransportCertsPublicSecret(context.Background(), client, *owner, ca, tt.extraCA)
 			if tt.wantErr {
 				require.Error(t, err, "Failed to reconcile")
 				return