Support a globally shared CA

NOTICE.txt

@@ -305,44 +305,6 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-ucfg@v0.8.4/
    limitations under the License.
 
 
---------------------------------------------------------------------------------
-Module  : github.com/fsnotify/fsnotify
-Version : v1.5.1
-Time    : 2021-08-24T19:33:57Z
-Licence : BSD-3-Clause
-
-Contents of probable licence file $GOMODCACHE/github.com/fsnotify/fsnotify@v1.5.1/LICENSE:
-
-Copyright (c) 2012 The Go Authors. All rights reserved.
-Copyright (c) 2012-2019 fsnotify Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Module  : github.com/ghodss/yaml
 Version : v1.0.0
@@ -6204,6 +6166,44 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Module  : github.com/fsnotify/fsnotify
+Version : v1.5.1
+Time    : 2021-08-24T19:33:57Z
+Licence : BSD-3-Clause
+
+Contents of probable licence file $GOMODCACHE/github.com/fsnotify/fsnotify@v1.5.1/LICENSE:
+
+Copyright (c) 2012 The Go Authors. All rights reserved.
+Copyright (c) 2012-2019 fsnotify Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Module  : github.com/go-logr/zapr
 Version : v1.2.0

cmd/manager/main.go

@@ -16,7 +16,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"github.com/go-logr/logr"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -83,6 +82,7 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/dev/portforward"
 	licensing "github.com/elastic/cloud-on-k8s/pkg/license"
 	"github.com/elastic/cloud-on-k8s/pkg/telemetry"
+	"github.com/elastic/cloud-on-k8s/pkg/utils/fs"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 	logconf "github.com/elastic/cloud-on-k8s/pkg/utils/log"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/metrics"
@@ -124,10 +124,6 @@ func Command() *cobra.Command {
 				if err := viper.ReadInConfig(); err != nil {
 					return fmt.Errorf("failed to read config file %s: %w", configFile, err)
 				}
-
-				if !viper.GetBool(operator.DisableConfigWatch) {
-					viper.WatchConfig()
-				}
 			}
 
 			logconf.ChangeVerbosity(viper.GetInt(logconf.FlagName))
@@ -144,6 +140,11 @@ func Command() *cobra.Command {
 		"Enables automatic port-forwarding "+
 			"(for dev use only as it exposes k8s resources on ephemeral ports to localhost)",
 	)
+	cmd.Flags().String(
+		operator.CAFlag,
+		"",
+		"Path to a CA certificate (tls.crt) and private key (tls.key) to be used for all managed resources. Effectively disables the CA rotation options.",
+	)
 	cmd.Flags().Duration(
 		operator.CACertRotateBeforeFlag,
 		certificates.DefaultRotateBefore,
@@ -316,27 +317,39 @@ func Command() *cobra.Command {
 
 func doRun(_ *cobra.Command, _ []string) error {
 	ctx := signals.SetupSignalHandler()
-	disableConfigWatch := viper.GetBool(operator.DisableConfigWatch)
 
-	// no config file to watch so start the operator directly
-	if configFile == "" || disableConfigWatch {
-		return startOperator(ctx)
+	// receive config/CA file update events over a channel
+	confUpdateChan := make(chan struct{}, 1)
+	var toWatch []string
+
+	// watch for config file changes
+	if !viper.GetBool(operator.DisableConfigWatch) && configFile != "" {
+		toWatch = append(toWatch, configFile)
 	}
 
-	// receive config file update events over a channel
-	confUpdateChan := make(chan struct{}, 1)
+	// watch for CA files if configured
+	caDir := viper.GetString(operator.CAFlag)
+	if caDir != "" {
+		toWatch = append(toWatch,
+			filepath.Join(caDir, certificates.KeyFileName),
+			filepath.Join(caDir, certificates.CertFileName),
+			filepath.Join(caDir, certificates.CAKeyFileName),
+			filepath.Join(caDir, certificates.CAFileName),
+		)
+	}
 
-	viper.OnConfigChange(func(evt fsnotify.Event) {
-		if evt.Op&fsnotify.Write == fsnotify.Write || evt.Op&fsnotify.Create == fsnotify.Create {
-			confUpdateChan <- struct{}{}
-		}
-	})
+	onConfChange := func(_ []string) {
+		confUpdateChan <- struct{}{}
+	}
+	watcher := fs.NewFileWatcher(ctx, toWatch, onConfChange, 15*time.Second)
+	go watcher.Run()
 
-	// start the operator in a goroutine
+	// set up channels and context for the operator
 	errChan := make(chan error, 1)
 	ctx, cancelFunc := context.WithCancel(ctx)
 	defer cancelFunc()
 
+	// start the operator in a goroutine
 	go func() {
 		err := startOperator(ctx)
 		if err != nil {
@@ -350,15 +363,12 @@ func doRun(_ *cobra.Command, _ []string) error {
 		select {
 		case err := <-errChan: // operator failed
 			log.Error(err, "Shutting down due to error")
-
 			return err
 		case <-ctx.Done(): // signal received
 			log.Info("Shutting down due to signal")
-
 			return nil
 		case <-confUpdateChan: // config file updated
 			log.Info("Shutting down to apply updated configuration")
-
 			return nil
 		}
 	}
@@ -501,6 +511,13 @@ func startOperator(ctx context.Context) error {
 		return err
 	}
 
+	// Retrieve shared CA if any
+	ca, err := readOptionalCA(viper.GetString(operator.CAFlag))
+	if err != nil {
+		log.Error(err, "Invalid CA")
+		return err
+	}
+
 	// Verify cert validity options
 	caCertValidity, caCertRotateBefore, err := validateCertExpirationFlags(operator.CACertValidityFlag, operator.CACertRotateBeforeFlag)
 	if err != nil {
@@ -562,6 +579,7 @@ func startOperator(ctx context.Context) error {
 		IPFamily:          ipFamily,
 		OperatorNamespace: operatorNamespace,
 		OperatorInfo:      operatorInfo,
+		CA:                ca,
 		CACertRotation: certificates.RotationParams{
 			Validity:     caCertValidity,
 			RotateBefore: caCertRotateBefore,
@@ -636,6 +654,13 @@ func startOperator(ctx context.Context) error {
 	}
 }
 
+func readOptionalCA(caDir string) (*certificates.CA, error) {
+	if caDir == "" {
+		return nil, nil
+	}
+	return certificates.BuildCAFromFile(caDir)
+}
+
 // asyncTasks schedules some tasks to be started when this instance of the operator is elected
 func asyncTasks(
 	mgr manager.Manager,

config/e2e/helm-operator-under-test.yaml

@@ -49,3 +49,15 @@ config:
 internal:
   createOperatorNamespace: false
   kubeVersion: {{ .KubernetesVersion }}
+
+# mount an empty secret for tests that exercise the shared CA functionality
+# tests can then update the secret contents as needed and reconfigure the operator
+volumes:
+  - name: ca-placeholder
+    secret:
+      secretName: eck-ca-{{.TestRun}}
+
+volumeMounts:
+  - name: ca-placeholder
+    mountPath: /tmp/ca-certs
+    readOnly: true

config/e2e/secrets.yaml

@@ -24,3 +24,12 @@ data:
   {{ $key }}: |
     {{ $value | b64enc }}
   {{ end }}
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "eck-ca-{{ .Context.TestRun }}"
+  namespace: {{ .Context.Operator.Namespace }}
+  labels:
+    test-run: {{ .Context.TestRun }}
+...

docs/reference/dependencies.asciidoc

@@ -23,7 +23,6 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://github.com/blang/semver[$$github.com/blang/semver/v4$$] | v4.0.0 | MIT
 | link:https://github.com/davecgh/go-spew[$$github.com/davecgh/go-spew$$] | v1.1.1 | ISC
 | link:https://github.com/elastic/go-ucfg[$$github.com/elastic/go-ucfg$$] | v0.8.4 | Apache-2.0
-| link:https://github.com/fsnotify/fsnotify[$$github.com/fsnotify/fsnotify$$] | v1.5.1 | BSD-3-Clause
 | link:https://github.com/ghodss/yaml[$$github.com/ghodss/yaml$$] | v1.0.0 | MIT
 | link:https://github.com/go-logr/logr[$$github.com/go-logr/logr$$] | v1.2.3 | Apache-2.0
 | link:https://github.com/go-test/deep[$$github.com/go-test/deep$$] | v1.0.8 | MIT
@@ -90,6 +89,7 @@ This page lists the third-party dependencies used to build {n}.
 | link:https://github.com/fatih/color[$$github.com/fatih/color$$] | v1.13.0 | MIT
 | link:https://github.com/fatih/structs[$$github.com/fatih/structs$$] | v1.1.0 | MIT
 | link:https://github.com/frankban/quicktest[$$github.com/frankban/quicktest$$] | v1.13.0 | MIT
+| link:https://github.com/fsnotify/fsnotify[$$github.com/fsnotify/fsnotify$$] | v1.5.1 | BSD-3-Clause
 | link:https://github.com/go-logr/zapr[$$github.com/go-logr/zapr$$] | v1.2.0 | Apache-2.0
 | link:https://github.com/gogo/protobuf[$$github.com/gogo/protobuf$$] | v1.3.2 | BSD-3-Clause
 | link:https://github.com/golang/groupcache[$$github.com/golang/groupcache$$] | v0.0.0-20210331224755-41bb18bfe9da | Apache-2.0

go.mod

@@ -7,7 +7,6 @@ require (
 	github.com/blang/semver/v4 v4.0.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/elastic/go-ucfg v0.8.4
-	github.com/fsnotify/fsnotify v1.5.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-logr/logr v1.2.3
 	github.com/go-test/deep v1.0.8
@@ -61,6 +60,7 @@ require (
 	github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect
 	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
 	github.com/fatih/color v1.13.0 // indirect
+	github.com/fsnotify/fsnotify v1.5.1 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

pkg/controller/agent/driver.go

@@ -103,6 +103,7 @@ func internalReconcile(params Params) *reconciler.Results {
 			Namer:                 Namer,
 			Labels:                NewLabels(params.Agent),
 			Services:              []corev1.Service{*svc},
+			GlobalCA:              params.OperatorParams.CA,
 			CACertRotation:        params.OperatorParams.CACertRotation,
 			CertRotation:          params.OperatorParams.CertRotation,
 			GarbageCollectSecrets: true,

pkg/controller/apmserver/controller.go

@@ -230,6 +230,7 @@ func (r *ReconcileApmServer) doReconcile(ctx context.Context, request reconcile.
 		Namer:                 Namer,
 		Labels:                NewLabels(as.Name),
 		Services:              []corev1.Service{*svc},
+		GlobalCA:              r.CA,
 		CACertRotation:        r.CACertRotation,
 		CertRotation:          r.CertRotation,
 		GarbageCollectSecrets: true,

pkg/controller/common/certificates/ca_reconcile.go

@@ -11,6 +11,8 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"time"
 
 	"github.com/pkg/errors"
@@ -22,6 +24,7 @@ import (
 
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/name"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler"
+	"github.com/elastic/cloud-on-k8s/pkg/utils/fs"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 )
 
@@ -232,6 +235,66 @@ func internalSecretForCA(
 	}, nil
 }
 
+func detectCAFileNames(path string) (string, string, error) {
+	files := map[string]bool{
+		CertFileName:  false,
+		KeyFileName:   false,
+		CAFileName:    false,
+		CAKeyFileName: false,
+	}
+	for f := range files {
+		exists, err := fs.FileExists(filepath.Join(path, f))
+		if err != nil {
+			return "", "", err
+		}
+		files[f] = exists
+	}
+	switch {
+	case (files[CertFileName] || files[KeyFileName]) && files[CAKeyFileName]:
+		return "", "", fmt.Errorf("both tls.* and ca.* files exist, configuration error")
+	case files[CAFileName] && files[CAKeyFileName]:
+		return filepath.Join(path, CAFileName), filepath.Join(CAKeyFileName), nil
+	case files[CertFileName] && files[KeyFileName]:
+		return filepath.Join(path, CertFileName), filepath.Join(path, KeyFileName), nil
+	}
+	return "", "", fmt.Errorf("no CA certificate files found: %+v", files)
+}
+
+func BuildCAFromFile(path string) (*CA, error) {
+	certFile, privateKeyFile, err := detectCAFileNames(path)
+	if err != nil {
+		return nil, err
+	}
+
+	bytes, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	certs, err := ParsePEMCerts(bytes)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Cannot parse PEM cert from %s", certFile)
+	}
+
+	if len(certs) == 0 {
+		return nil, fmt.Errorf("PEM %s file did not contain any certificates", certFile)
+	}
+
+	if len(certs) > 1 {
+		return nil, fmt.Errorf("more than one certificate in PEM file %s", certFile)
+	}
+	cert := certs[0]
+
+	privateKeyBytes, err := ioutil.ReadFile(privateKeyFile)
+	if err != nil {
+		return nil, err
+	}
+	privateKey, err := ParsePEMPrivateKey(privateKeyBytes)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Cannot parse private key from PEM file %s", privateKeyFile)
+	}
+	return NewCA(privateKey, cert), nil
+}
+
 // BuildCAFromSecret parses the given secret into a CA.
 // It returns nil if the secrets could not be parsed into a CA.
 func BuildCAFromSecret(caInternalSecret corev1.Secret) *CA {