Allow custom certificates on the transport layer (#6727)

Introduce a new attribute transport.tls.certificateAuthorities.configMapName which expects a config map with a ca.crt file that can contain one or more CAs in PEM format. Alternative name suggestion transport.tls.trust

Concatenate these additional CAs with the CA issued by ECK in the internal and public secrets. This ensures that all existing mechanisms including remote cluster support seamlessly pick up the additional trust. It also allows transitioning without downtime from self-issued certificates to third-party issued certificates.

Adds a documentation section to illustrate how transport certificates can be configured with cert-manager csi-driver (arguably could also be a recipe but not one we auto-test because of the additional infrastructure requirements)

config/crds/v1/all-crds.yaml

@@ -5178,6 +5178,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a reference to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

config/crds/v1/bases/elasticsearch.k8s.elastic.co_elasticsearches.yaml

@@ -9600,6 +9600,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a reference to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

deploy/eck-operator/charts/eck-operator-crds/templates/all-crds.yaml

@@ -5214,6 +5214,15 @@ spec:
                             description: SecretName is the name of the secret.
                             type: string
                         type: object
+                      certificateAuthorities:
+                        description: CertificateAuthorities is a reference to a config
+                          map that contains one or more x509 certificates for trusted
+                          authorities in PEM format. The certificates need to be in
+                          a file called `ca.crt`.
+                        properties:
+                          configMapName:
+                            type: string
+                        type: object
                       otherNameSuffix:
                         description: 'OtherNameSuffix when defined will be prefixed
                           with the Pod name and used as the common name, and the first

docs/orchestrating-elastic-stack-applications/elasticsearch/remote-clusters.asciidoc

@@ -79,14 +79,14 @@ NOTE: CA certificates are automatically rotated after one year by default. You c
 
 If `cluster-two` is also managed by an ECK instance, proceed as follows:
 
-. Create a secret with the CA certificate you just extracted:
+. Create a config map with the CA certificate you just extracted:
 +
 [source,sh]
 ----
-kubectl create secret generic remote-certs --from-file=remote.ca.crt
+kubectl create configmap remote-certs --from-file=ca.crt=remote.ca.crt
 ----
 
-. Use this secret to configure `cluster-one`'s CA as a trusted CA in `cluster-two`:
+. Use this config map to configure `cluster-one`'s CA as a trusted CA in `cluster-two`:
 +
 [source,yaml,subs="attributes"]
 ----
@@ -95,23 +95,13 @@ kind: Elasticsearch
 metadata:
   name: cluster-two
 spec:
+  transport:
+    tls:
+      certificateAuthorities:
+        configMapName: remote-certs
   nodeSets:
-  - config:
-      xpack.security.transport.ssl.certificate_authorities:
-      - /usr/share/elasticsearch/config/other/remote.ca.crt
-    count: 3
+  - count: 3
     name: default
-    podTemplate:
-      spec:
-        containers:
-        - name: elasticsearch
-          volumeMounts:
-          - mountPath: /usr/share/elasticsearch/config/other
-            name: remote-certs
-        volumes:
-        - name: remote-certs
-          secret:
-            secretName: remote-certs
   version: {version}
 ----
 

docs/orchestrating-elastic-stack-applications/elasticsearch/reserved-settings.asciidoc

@@ -24,9 +24,7 @@ The following Elasticsearch settings are managed by ECK:
 * `xpack.security.http.ssl.certificate`
 * `xpack.security.http.ssl.enabled`
 * `xpack.security.http.ssl.key`
-* `xpack.security.transport.ssl.certificate`
 * `xpack.security.transport.ssl.enabled`
-* `xpack.security.transport.ssl.key`
 * `xpack.security.transport.ssl.verification_mode`
 
 The following Elasticsearch settings are not supported by ECK:

docs/orchestrating-elastic-stack-applications/elasticsearch/transport-settings.asciidoc

@@ -68,3 +68,95 @@ spec:
   - name: default
     count: 3
 ----
+
+== Issue node transport certificates with third-party tools
+
+When following the instructions in <<{p}-transport-ca>> the issuance of certificates is orchestrated by the ECK operator and the operator needs access to the CAs private key.
+If this is undesirable it is also possible to configure node transport certificates without involving the ECK operator. The following two pre-requisites apply:
+
+1. The tooling used must be able to issue individual certificates for each Elasticsearch node and dynamically add or remove certificates as the cluster scales up and down.
+2. The ECK operator must be configured to be aware of the CA in use for the <<{p}-remote-clusters-connect-external,remote cluster>> support to work.
+
+The following example configuration using link:https://cert-manager.io/docs/projects/csi-driver/[cert-manager csi-driver] and link:https://cert-manager.io/docs/projects/trust-manager/[trust-manager] meets these two requirements:
+
+[source,yaml,subs="attributes"]
+----
+apiVersion: elasticsearch.k8s.elastic.co/{eck_crd_version}
+kind: Elasticsearch
+metadata:
+  name: es
+spec:
+  version: {version}
+  transport:
+    tls:
+      certificateAuthorities:
+        configMapName: trust
+  nodeSets:
+  - name: default
+    count: 3
+    config:
+      xpack.security.transport.ssl.key: /usr/share/elasticsearch/config/cert-manager-certs/tls.key
+      xpack.security.transport.ssl.certificate: /usr/share/elasticsearch/config/cert-manager-certs/tls.crt
+    podTemplate:
+      spec:
+        containers:
+        - name: elasticsearch
+          volumeMounts:
+          - name: transport-certs
+            mountPath: /usr/share/elasticsearch/config/cert-manager-certs
+        volumes:
+        - name: transport-certs
+          csi:
+            driver: csi.cert-manager.io
+            readOnly: true
+            volumeAttributes:
+              csi.cert-manager.io/issuer-name: ca-cluster-issuer
+              csi.cert-manager.io/issuer-kind: ClusterIssuer
+              csi.cert-manager.io/dns-names: "${POD_NAME}.${POD_NAMESPACE}.svc.cluster.local"
+----
+
+The example assumes that a `ClusterIssuer` by the name of `ca-cluster-issuer` exists and a PEM encoded version of the CA certificate is available in a ConfigMap (in the example named `trust`).  The CA certificate must be in a file called  `ca.crt` inside the ConfigMap in the same namespace as the Elasticsearch resource.
+
+The following manifest is only provided to illustrate how these certificates can be configured in principle, using the trust-manager Bundle resource and cert-manager provisioned certificates:
+
+[source,yaml]
+----
+apiVersion: trust.cert-manager.io/v1alpha1
+kind: Bundle
+metadata:
+  name: trust
+spec:
+  sources:
+  - secret:
+      name: "root-ca-secret"
+      key: "tls.crt"
+  target:
+    configMap:
+      key: "ca.crt"
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: selfsigned-ca
+  namespace: cert-manager
+spec:
+  isCA: true
+  commonName: selfsigned-ca
+  secretName: root-ca-secret
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  issuerRef:
+    name: selfsigned-issuer
+    kind: ClusterIssuer
+    group: cert-manager.io
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: ca-cluster-issuer
+spec:
+  ca:
+    secretName: root-ca-secret
+...
+----

docs/reference/api-docs.asciidoc

@@ -456,6 +456,23 @@ Config represents untyped YAML configuration.
 
 
 
+[id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref"]
+=== ConfigMapRef 
+
+ConfigMapRef is a reference to a config map that exists in the same namespace as the referring resource.
+
+.Appears In:
+****
+- xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-elasticsearch-v1-transporttlsoptions[$$TransportTLSOptions$$]
+****
+
+[cols="25a,75a", options="header"]
+|===
+| Field | Description
+| *`configMapName`* __string__ | 
+|===
+
+
 [id="{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configsource"]
 === ConfigSource 
 
@@ -1391,6 +1408,7 @@ TransportConfig holds the transport layer settings for Elasticsearch.
 | *`subjectAltNames`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-subjectalternativename[$$SubjectAlternativeName$$] array__ | SubjectAlternativeNames is a list of SANs to include in the generated node transport TLS certificates.
 | *`certificate`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-secretref[$$SecretRef$$]__ | Certificate is a reference to a Kubernetes secret that contains the CA certificate and private key for generating node certificates. The referenced secret should contain the following: 
  - `ca.crt`: The CA certificate in PEM format. - `ca.key`: The private key for the CA certificate in PEM format.
+| *`certificateAuthorities`* __xref:{anchor_prefix}-github-com-elastic-cloud-on-k8s-v2-pkg-apis-common-v1-configmapref[$$ConfigMapRef$$]__ | CertificateAuthorities is a reference to a config map that contains one or more x509 certificates for trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
 |===
 
 

pkg/apis/common/v1/common.go

@@ -45,6 +45,15 @@ func (ds DeploymentStatus) IsDegraded(prev DeploymentStatus) bool {
 	return prev.Health == GreenHealth && ds.Health != GreenHealth
 }
 
+// ConfigMapRef is a reference to a config map that exists in the same namespace as the referring resource.
+type ConfigMapRef struct {
+	ConfigMapName string `json:"configMapName,omitempty"`
+}
+
+func (c ConfigMapRef) IsDefined() bool {
+	return len(c.ConfigMapName) > 0
+}
+
 // SecretRef is a reference to a secret that exists in the same namespace.
 type SecretRef struct {
 	// SecretName is the name of the secret.

pkg/apis/elasticsearch/v1/elasticsearch_types.go

@@ -172,6 +172,9 @@ type TransportTLSOptions struct {
 	// - `ca.crt`: The CA certificate in PEM format.
 	// - `ca.key`: The private key for the CA certificate in PEM format.
 	Certificate commonv1.SecretRef `json:"certificate,omitempty"`
+	// CertificateAuthorities is a reference to a config map that contains one or more x509 certificates for
+	// trusted authorities in PEM format. The certificates need to be in a file called `ca.crt`.
+	CertificateAuthorities commonv1.ConfigMapRef `json:"certificateAuthorities,omitempty"`
 }
 
 func (tto TransportTLSOptions) UserDefinedCA() bool {

pkg/apis/elasticsearch/v1/fields.go

@@ -65,8 +65,6 @@ var UnsupportedSettings = []string{
 	XPackSecurityHttpSslCertificate,
 	XPackSecurityHttpSslEnabled,
 	XPackSecurityHttpSslKey,
-	XPackSecurityTransportSslCertificate,
 	XPackSecurityTransportSslEnabled,
-	XPackSecurityTransportSslKey,
 	XPackSecurityTransportSslVerificationMode,
 }

pkg/controller/common/watches/state.go

@@ -7,6 +7,7 @@ package watches
 // NewDynamicWatches creates an initialized DynamicWatches container.
 func NewDynamicWatches() DynamicWatches {
 	return DynamicWatches{
+		ConfigMaps:          NewDynamicEnqueueRequest(),
 		Secrets:             NewDynamicEnqueueRequest(),
 		Services:            NewDynamicEnqueueRequest(),
 		Pods:                NewDynamicEnqueueRequest(),
@@ -17,6 +18,7 @@ func NewDynamicWatches() DynamicWatches {
 // DynamicWatches contains stateful dynamic watches. Intended as facility to pass around stateful dynamic watches and
 // give each of them an identity.
 type DynamicWatches struct {
+	ConfigMaps          *DynamicEnqueueRequest
 	Secrets             *DynamicEnqueueRequest
 	Services            *DynamicEnqueueRequest
 	Pods                *DynamicEnqueueRequest

pkg/controller/elasticsearch/certificates/reconcile.go

@@ -100,6 +100,14 @@ func ReconcileTransport(
 	// label certificates secrets with the cluster name
 	certsLabels := label.NewLabels(k8s.ExtractNamespacedName(&es))
 
+	// Maybe retrieve user defined additional trusted CAs from a config map.
+	// They will be concatenated with the ECK issued CA and distributed through the same transport secrets.
+	additionalCAs, err := transport.ReconcileAdditionalCAs(ctx, driver.K8sClient(), es, driver.DynamicWatches())
+	if err != nil {
+		driver.Recorder().Eventf(&es, corev1.EventTypeWarning, events.EventReasonUnexpected, err.Error())
+		return results.WithError(err)
+	}
+
 	// reconcile transport CA and certs
 	transportCA, err := transport.ReconcileOrRetrieveCA(
 		ctx,
@@ -120,7 +128,7 @@ func ReconcileTransport(
 	)
 
 	// reconcile transport public certs secret
-	if err := transport.ReconcileTransportCertsPublicSecret(ctx, driver.K8sClient(), es, transportCA); err != nil {
+	if err := transport.ReconcileTransportCertsPublicSecret(ctx, driver.K8sClient(), es, transportCA, additionalCAs); err != nil {
 		return results.WithError(err)
 	}
 
@@ -129,6 +137,7 @@ func ReconcileTransport(
 		ctx,
 		driver.K8sClient(),
 		transportCA,
+		additionalCAs,
 		es,
 		certRotation,
 	)

pkg/controller/elasticsearch/certificates/transport/public_secret.go

@@ -5,6 +5,7 @@
 package transport
 
 import (
+	"bytes"
 	"context"
 
 	corev1 "k8s.io/api/core/v1"
@@ -24,6 +25,7 @@ func ReconcileTransportCertsPublicSecret(
 	c k8s.Client,
 	es esv1.Elasticsearch,
 	ca *certificates.CA,
+	additionalCAs []byte,
 ) error {
 	esNSN := k8s.ExtractNamespacedName(&es)
 	meta := k8s.ToObjectMeta(PublicCertsSecretRef(esNSN))
@@ -32,7 +34,7 @@ func ReconcileTransportCertsPublicSecret(
 	expected := corev1.Secret{
 		ObjectMeta: meta,
 		Data: map[string][]byte{
-			certificates.CAFileName: certificates.EncodePEMCert(ca.Cert.Raw),
+			certificates.CAFileName: bytes.Join([][]byte{certificates.EncodePEMCert(ca.Cert.Raw), additionalCAs}, nil),
 		},
 	}