Allow custom certificates on the transport layer

config/crds/v1/all-crds.yaml

@@ -5178,6 +5178,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a references to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml

@@ -9600,6 +9600,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a references to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -5214,6 +5214,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a references to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

docs/orchestrating-elastic-stack-applications/elasticsearch/remote-clusters.asciidoc

@@ -79,14 +79,14 @@ NOTE: CA certificates are automatically rotated after one year by default. You c
 
 If `cluster-two` is also managed by an ECK instance, proceed as follows:
 
-. Create a secret with the CA certificate you just extracted:
+. Create a config map with the CA certificate you just extracted:
 +
 [source,sh]
 ----
-kubectl create secret generic remote-certs --from-file=remote.ca.crt
+kubectl create configmap remote-certs --from-file=ca.crt=remote.ca.crt
 ----
 
-. Use this secret to configure `cluster-one`'s CA as a trusted CA in `cluster-two`:
+. Use this config map to configure `cluster-one`'s CA as a trusted CA in `cluster-two`:
 +
 [source,yaml,subs="attributes"]
 ----
@@ -95,23 +95,13 @@ kind: Elasticsearch
 metadata:
   name: cluster-two
 spec:
+  transport:
+    tls:
+      certificateAuthorities:
+        configMapName: remote-certs
   nodeSets:
-  - config:
-      xpack.security.transport.ssl.certificate_authorities:
-      - /usr/share/elasticsearch/config/other/remote.ca.crt
-    count: 3
+  - count: 3
     name: default
-    podTemplate:
-      spec:
-        containers:
-        - name: elasticsearch
-          volumeMounts:
-          - mountPath: /usr/share/elasticsearch/config/other
-            name: remote-certs
-        volumes:
-        - name: remote-certs
-          secret:
-            secretName: remote-certs
   version: {version}
 ----
 

docs/orchestrating-elastic-stack-applications/elasticsearch/reserved-settings.asciidoc

@@ -24,9 +24,7 @@ The following Elasticsearch settings are managed by ECK:
 * `xpack.security.http.ssl.certificate`
 * `xpack.security.http.ssl.enabled`
 * `xpack.security.http.ssl.key`
-* `xpack.security.transport.ssl.certificate`
 * `xpack.security.transport.ssl.enabled`
-* `xpack.security.transport.ssl.key`
 * `xpack.security.transport.ssl.verification_mode`
 
 The following Elasticsearch settings are not supported by ECK:

docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc

@@ -68,3 +68,95 @@ spec:
   - name: default
     count: 3
 ----
+
+== Issue node transport certificates with third-party tools
+
+When following the instructions in <<{p}-transport-ca>> the issuance of certificates is orchestrated by the ECK operator and the operator needs access to the CAs private key.
+If this is undesirable it is also possible to configure node transport certificates without involving the ECK operator. The following two pre-requisites apply:
+
+1. The tooling used must be able to issue individual certificates for each Elasticsearch node and dynamically add or remove certificates as the cluster scales up and down.
+2. The ECK operator must be configured to be aware of the CA in use for the <<{p}-remote-clusters-connect-external,remote cluster>> support to work.
+
+The following example configuration using link:https://cert-manager.io/docs/projects/csi-driver/[cert-manager csi-driver] and link:https://cert-manager.io/docs/projects/trust-manager/[trust-manager] meets these two requirements:
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: elasticsearch.k8s.elastic.co/{eck_crd_version}
+kind: Elasticsearch
+metadata:
+  name: es
+spec:
+  version: {version}
+  transport:
+    tls:
+      certificateAuthorities:
+        configMapName: trust
+  nodeSets:
+  - name: default
+    count: 3
+    config:
+      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/cert-manager-certs/tls.key
+      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/cert-manager-certs/tls.crt
+    podTemplate:
+      spec:
+        containers:
+        - name: elasticsearch
+          volumeMounts:
+          - name: transport-certs
+            mountPath: /usr/share/elasticsearch/config/cert-manager-certs
+        volumes:
+        - name: transport-certs
+          csi:
+            driver: csi.cert-manager.io
+            readOnly: true
+            volumeAttributes:
+              csi.cert-manager.io/issuer-name: ca-cluster-issuer
+              csi.cert-manager.io/issuer-kind: ClusterIssuer
+              csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local"
+----
+
+The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and a PEM encoded version of the CA certificate is available in a ConfigMap (in the example named `trust`).  The CA certificate must be in a file called  `ca.crt` inside the ConfigMap in the same namespace as the Elasticsearch resource.
+
+The following manifest is only provided to illustrate how these certificates can be configured in principle, using the trust-manager Bundle resource and cert-manager provisioned certificates:
+
+[source,yaml]
+----
+apiVersion: trust.cert-manager.io/v1alpha1
+kind: Bundle
+metadata:
+  name: trust
+spec:
+  sources:
+  - secret:
+      name: "root-ca-secret"
+      key: "tls.crt"
+  target:
+    configMap:
+      key: "ca.crt"
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: selfsigned-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: selfsigned-ca
+  secretName: root-ca-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: ca-cluster-issuer
+spec:
+  ca:
+    secretName: root-ca-secret
+...
+----

docs/reference/api-docs.asciidoc

@@ -454,6 +454,23 @@ Config represents untyped YAML configuration.
 
 
 
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref"]
+=== ConfigMapRef 
+
+ConfigMapRef is a reference to a config map that exists in the same namespace as the referring resource.
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transporttlsoptions[$$TransportTLSOptions$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`configMapName`* __string__ | 
+|===
+
+
 [id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configsource"]
 === ConfigSource 
 
@@ -1383,6 +1400,7 @@ TransportConfig holds the transport layer settings for Elasticsearch.
 | *`subjectAltNames`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-subjectalternativename[$$SubjectAlternativeName$$] array__ | SubjectAlternativeNames is a list of SANs to include in the generated node transport TLS certificates.
 | *`certificate`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-secretref[$$SecretRef$$]__ | Certificate is a reference to a Kubernetes secret that contains the CA certificate and private key for generating node certificates. The referenced secret should contain the following: 
  - `ca.crt`: The CA certificate in PEM format. - `ca.key`: The private key for the CA certificate in PEM format.
+| *`certificateAuthorities`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref[$$ConfigMapRef$$]__ | CertificateAuthorities is a references to a config map that contains one or more x509 certificates for trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
 |===
 
 

pkg/apis/common/v1/common.go

@@ -45,6 +45,15 @@ func (ds DeploymentStatus) IsDegraded(prev DeploymentStatus) bool {
 	return prev.Health == GreenHealth && ds.Health != GreenHealth
 }
 
+// ConfigMapRef is a reference to a config map that exists in the same namespace as the referring resource.
+type ConfigMapRef struct {
+	ConfigMapName string `json:"configMapName,omitempty"`
+}
+
+func (c ConfigMapRef) IsDefined() bool {
+	return len(c.ConfigMapName) > 0
+}
+
 // SecretRef is a reference to a secret that exists in the same namespace.
 type SecretRef struct {
 	// SecretName is the name of the secret.

pkg/apis/elasticsearch/v1/elasticsearch_types.go

@@ -172,6 +172,9 @@ type TransportTLSOptions struct {
 	// - `ca.crt`: The CA certificate in PEM format.
 	// - `ca.key`: The private key for the CA certificate in PEM format.
 	Certificate commonv1.SecretRef `json:"certificate,omitempty"`
+	// CertificateAuthorities is a references to a config map that contains one or more x509 certificates for
+	// trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
+	CertificateAuthorities commonv1.ConfigMapRef `json:"certificateAuthorities,omitempty"`
 }
 
 func (tto TransportTLSOptions) UserDefinedCA() bool {

pkg/apis/elasticsearch/v1/fields.go

@@ -65,8 +65,6 @@ var UnsupportedSettings = []string{
 	XPackSecurityHttpSslCertificate,
 	XPackSecurityHttpSslEnabled,
 	XPackSecurityHttpSslKey,
-	XPackSecurityTransportSslCertificate,
 	XPackSecurityTransportSslEnabled,
-	XPackSecurityTransportSslKey,
 	XPackSecurityTransportSslVerificationMode,
 }

pkg/controller/common/watches/state.go

@@ -7,6 +7,7 @@ package watches
 // NewDynamicWatches creates an initialized DynamicWatches container.
 func NewDynamicWatches() DynamicWatches {
 	return DynamicWatches{
+		ConfigMaps:          NewDynamicEnqueueRequest(),
 		Secrets:             NewDynamicEnqueueRequest(),
 		Services:            NewDynamicEnqueueRequest(),
 		Pods:                NewDynamicEnqueueRequest(),
@@ -17,6 +18,7 @@ func NewDynamicWatches() DynamicWatches {
 // DynamicWatches contains stateful dynamic watches. Intended as facility to pass around stateful dynamic watches and
 // give each of them an identity.
 type DynamicWatches struct {
+	ConfigMaps          *DynamicEnqueueRequest
 	Secrets             *DynamicEnqueueRequest
 	Services            *DynamicEnqueueRequest
 	Pods                *DynamicEnqueueRequest

pkg/controller/elasticsearch/certificates/reconcile.go

@@ -100,6 +100,14 @@ func ReconcileTransport(
 	// label certificates secrets with the cluster name
 	certsLabels := label.NewLabels(k8s.ExtractNamespacedName(&es))
 
+	// Maybe retrieve user defined additional trusted CAs from a config map.
+	// They will be concatenated with the ECK issued CA and distributed through the same transport secrets.
+	additionalCAs, err := transport.ReconcileAdditionalCAs(ctx, driver.K8sClient(), es, driver.DynamicWatches())
+	if err != nil {
+		driver.Recorder().Eventf(&es, corev1.EventTypeWarning, events.EventReasonUnexpected, err.Error())
+		return results.WithError(err)
+	}
+
 	// reconcile transport CA and certs
 	transportCA, err := transport.ReconcileOrRetrieveCA(
 		ctx,
@@ -120,7 +128,7 @@ func ReconcileTransport(
 	)
 
 	// reconcile transport public certs secret
-	if err := transport.ReconcileTransportCertsPublicSecret(ctx, driver.K8sClient(), es, transportCA); err != nil {
+	if err := transport.ReconcileTransportCertsPublicSecret(ctx, driver.K8sClient(), es, transportCA, additionalCAs); err != nil {
 		return results.WithError(err)
 	}
 
@@ -129,6 +137,7 @@ func ReconcileTransport(
 		ctx,
 		driver.K8sClient(),
 		transportCA,
+		additionalCAs,
 		es,
 		certRotation,
 	)

pkg/controller/elasticsearch/certificates/transport/public_secret.go

@@ -5,6 +5,7 @@
 package transport
 
 import (
+	"bytes"
 	"context"
 
 	corev1 "k8s.io/api/core/v1"
@@ -24,6 +25,7 @@ func ReconcileTransportCertsPublicSecret(
 	c k8s.Client,
 	es esv1.Elasticsearch,
 	ca *certificates.CA,
+	additionalCAs []byte,
 ) error {
 	esNSN := k8s.ExtractNamespacedName(&es)
 	meta := k8s.ToObjectMeta(PublicCertsSecretRef(esNSN))
@@ -32,7 +34,7 @@ func ReconcileTransportCertsPublicSecret(
 	expected := corev1.Secret{
 		ObjectMeta: meta,
 		Data: map[string][]byte{
-			certificates.CAFileName: certificates.EncodePEMCert(ca.Cert.Raw),
+			certificates.CAFileName: bytes.Join([][]byte{certificates.EncodePEMCert(ca.Cert.Raw), additionalCAs}, nil),
 		},
 	}