Reorganization and Addition: Move TLS types and config out of the out…

…puts and support server options. (#7054)

* Reorganization and Addition: Move TLS types and config out of the outputs and support server options.

When working on the TLS TCP it was a bit strange to import a
package coming from the outputs; this commit addresses a few things:

- Move the `outputs/tls.go` and `transport/tls.go` into the common
under the transport folder.
- Add shims to make sure we keep backward compatibility on anything that
could be using theses classes.
- Extract common logic code to be reusable.
- Add inverse mapper for TLSVersion and tlsCiphersuite, to give a uint
and get the human string.
- Add a new `ServerConfig` config struct.

*This is a light refactoring, mostly moving code and adding a few
tests.

Fixes: #6079

* Adding: Developer changelog

* rename client_authentification to client_authentication

I think my french influence slipped on that one.

* authenfitication -> authentication

CHANGELOG-developer.asciidoc

@@ -20,6 +20,7 @@ The list below covers the major changes between 6.3.0 and master only.
 ==== Breaking changes
 
 - The beat.Pipeline is now passed to cfgfile.RunnerFactory. Beats using libbeat for module reloading or autodiscovery need to be adapted. {pull}7018[7017]
+- Moving of TLS helper functions and structs from `output/tls` to `tlscommon`. {pull}7054[7054]
 
 
 ==== Added

libbeat/common/transport/tlscommon/ca_test.key

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAv8IiJDAIDl+roQOWe+oSq46Nyuu9R+Iis0V1i6M7zA6Qijbx
+CSZ64cCFYQfKheRYQSZRstHPHSUM1gSvUih/sqZqsiNMYDbb9j7geMDvls4c7rsH
+x7xImD7nCrEVWkiapGIhkW6SOtVo18Zmw89FUuDFhoRmMHcQ+7AtM4uUNPkSqKcX
+vzG093SU0oNdIBdw5PzoQlvBh5DL0iRYC6y22cwJyjWTUEB5vTjOTDxiFzsovRtj
+pdjzSZACXyW68b99icLzmxzLvsZ7w8tFJ8uOPQAVxwg6SmMUorURv48sBjfVfN48
+7OjH3d+51ozNJjP1MmKoN2BoE8pWq0jdhOWhDQH+pRiRjfMuL+yvcIJ2pxdOv0F3
+KBkng7qEgEUA8cqaFnawDA7O3a20SeDFWSQtN6LsFjT7EDMzNkML1pJjbGK24QFC
+IOOvCJtaccuREN1OfbN1yhTz3VErbJttwO6j2KueasPHXU3qLu2FKOlsXbPy1XMu
+LYZgv8Zprcbs4KhQ3/A7/RO1cakxWlRwta63mUIM2xLIMIgRSR+DSZ5dJaDNO6i4
+9eIGQXRxDb9dxA2hoCcoTv7PJKyOpNb5vyxMXJGY7H5j1jEEcqEeuI5uvuUwugQG
+tsl1eFLXIeQLerOHEQoS6wMv0fHBtZOVCHu8CCrnt/ag7kn39nkwNofLovECAwEA
+AQKCAgA7hRB/1wDJJVzqb2ioMbF12pucXquzwjcvGeIwY4xN/D9VB1StmGoP5GgC
+BB8SjBvwrOoy7PiyfSuMyot4nuV0GD+J53bvble8CSw3jvtO/c7xMtBpaMHHr86a
+/Pg5u8t0NplgwMdWx6LxRr3jDVThMq9c33+wj2SQGtEM7Mgl4SGvg53VVKJtJJyE
+8w1Wxq/eA7o7zqs1XvZE1c8WYJeo5rIrN5HwGPMwjo9KDnwL5erxN60obzykmrSB
+v/5UxzE6L27ZuIhtQMJttYxTm9Ucjgg0bRNav4JKNpW5tcDedTootfqHNoHDFoxi
+UfXjY8E50HGSLrRfYDCinc1UUMo568Ed9vRPOBSfw9FAZy4iExifmfHJsn8Bepse
+xvYQfsYJpEsKoxzTTD7yLZALJEu18+8AHgYG6jFkvIlOUUjUKHiOyU5UlFErHk/P
+W2n9FZPzSTnZQ2J06Rwmj2ILZ86kXIYoL8kEJSYTCG4TQ6KX4oeJq8v4yVHf+SiD
+ZiYFWLAZbZQ46lL/7+dyy3rhLErm57DgYhJL/BqLys0GZdaazh12AcDcLjSQ6Yoh
+xQYOogq+6xB4k8mqMkNmln5JWdhzFGAzkhClnCToYpvPK8KTg3a0cLV7X1wLlyh9
+Nr0kGATrUr2bHzBZazhwMkSXh+JUDZhyK0ZflqySQX8lQbMooQKCAQEA5ZVySenZ
+qfRNHdcdjIf/J7/vu9cDnPAqszbGpt/GeLD3yag8zTUnTh8ZjFhQ3LH4SQ/4TdmF
+37PsuNIzlay1TJ2b6lf0XoDG9DgbW3PpuRSVy2QIse7p6lsyNISn6bIJR1XSr9aP
+pbgiQK9svq+QN0rSWSsQEDZB9rTNC+VcMY0r4043MxGFwGauiSoARmu6yqD3y/3q
+ah3bz1UTZpUbnlO6PHT2nE+pV+YVHNz/MfprEFc+Ob9vCm6oCEhQyyAnOjcFxDjV
+6J2uxn8MhDjvGOsJ8OfJt9UDhVBbzJXBfOZXO7bLDbWMzTfaa7BcQRaNkOY+ZPC/
+tW62E12hhxlHfQKCAQEA1dKC+LXFmQp36Dp1IrPEvU+AFF67MnxQErKptaCcGCo0
+A/udpSC3ivja5dPxJOM+wF0Vz3601biJUhI8Sar+P+V67dLrK/uY30Aq9GNrjtTj
+sDqZejqvJak+nHa+CHe8RfkMlrTs/bgTSdQ0Go4k7+pH+Vi1pVnE07PQT8n772JY
+ibLrkx54EUWqhh0+/q8MHd7pdNEYGhfft54GddZG6Tnmg4/PDyLcF9+TL86sV3Hv
+uV6ftGVjE/Jrer3RCvGz28iYCy+pXLtg6xt768iI0bTDL5A9EopLiONRVu7hJJf5
+nYTmvQdjbVsfm7a9o/UxG3jOkgIy5W3haCVOFt2rhQKCAQBfVXWF99No3YeAUql0
+h6yOhwc3yws3CgvRK3fGJ7o0t9fNJ01IMUBHEmb7fljlrAlb3YPQX/lVcVNlU/QT
+vQnz7Kan4yoYbAUxuHKzwShWsJObR8jMilcb+A6a/FL1mfZ8Zsj8N26i9BlVHwNb
+E3AhZbJ/UIB1GvK9TUqwG+fys5p74yjMzgPqZzkmwAgpNeb06W68iI3kzs1OBRfv
+Sw+S6VW2cSNOuU2qsGIoACUATepTeMbgF/w2Kskf11elYY6of9ynJKq+02uWBX/f
+D/1JLaCNJtL+wTebDklwZOdZxBSJOViMMs1rEjxi53MHnCPg/Zr/M3GIF5cH56OB
+hB/JAoIBAQCt8/4zYoYoFJkqZ+yF1+R18yiK6eq3juUB4TIqHkj/a843c0t0XKKV
+wBEtqvhi/zE9BD3LOhTaTrABAe7kK+V+jC4vL0m91YkwDx8jBYMqh03ZQEM+amG1
+bPQQDJZbgzW7Y3r3XKf1XfzrMmVVOVEZkesOEzpsFBUJ+h692uBIhyTqmZIHdWFP
+A/NP+pkWT8i2wHQDYlyOVd/enQQ6d6Hm+gDsBWH5uW1/SpeO7D/PQFU75JxfAaDS
+SIViLOzVT3/4jUAM0bCiTZryisCNOO7+VGX62wikfbgn3G9/HwYxZCZiHQ4uuMUN
+4XVclBXCPqa959F+faV0e6lGthrKhXqVAoIBAGAVqGQrexKADcE3TKbOBAaOi8vo
+9HcTraZWOBY8QSP5xQZRey3L3sNrCTmT8L8fNmvXMMBoK9Lm51EYS8vgedUvlII9
+rC19IT0TG39AdFQH4/rWfcF9eqpneItPWuCRM3UokfeqDkS+4pBEGVOhI+dNr0oJ
+APXpue6CgbD9xLvNAvdn0/PgmD0tV4HO6VUbJ9W3yFE1j+m1vNHVwk36nEdaL1aC
+x7DTAiMGqrcTDr7DXwOImhPLrSWkLPxmIp+GD4831cmJqSSp/Lg/6OHa5fFZEJg7
+gkY+tjXMvUbuSx4lrOW6SY9LIxi7xTcRdfnd9g6z/G7IyGvXTevXDpopASo=
+-----END RSA PRIVATE KEY-----

libbeat/common/transport/tlscommon/ca_test.pem

@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFTDCCAzSgAwIBAgIRAOAMlgVxz4G+Zj/EtBTvpg4wDQYJKoZIhvcNAQENBQAw
+LzELMAkGA1UEBhMCVVMxEDAOBgNVBAoTB2VsYXN0aWMxDjAMBgNVBAsTBWJlYXRz
+MB4XDTE3MDUxODIwMzI1MVoXDTI3MDUxODIwMzI1MVowLzELMAkGA1UEBhMCVVMx
+EDAOBgNVBAoTB2VsYXN0aWMxDjAMBgNVBAsTBWJlYXRzMIICIjANBgkqhkiG9w0B
+AQEFAAOCAg8AMIICCgKCAgEAv8IiJDAIDl+roQOWe+oSq46Nyuu9R+Iis0V1i6M7
+zA6QijbxCSZ64cCFYQfKheRYQSZRstHPHSUM1gSvUih/sqZqsiNMYDbb9j7geMDv
+ls4c7rsHx7xImD7nCrEVWkiapGIhkW6SOtVo18Zmw89FUuDFhoRmMHcQ+7AtM4uU
+NPkSqKcXvzG093SU0oNdIBdw5PzoQlvBh5DL0iRYC6y22cwJyjWTUEB5vTjOTDxi
+FzsovRtjpdjzSZACXyW68b99icLzmxzLvsZ7w8tFJ8uOPQAVxwg6SmMUorURv48s
+BjfVfN487OjH3d+51ozNJjP1MmKoN2BoE8pWq0jdhOWhDQH+pRiRjfMuL+yvcIJ2
+pxdOv0F3KBkng7qEgEUA8cqaFnawDA7O3a20SeDFWSQtN6LsFjT7EDMzNkML1pJj
+bGK24QFCIOOvCJtaccuREN1OfbN1yhTz3VErbJttwO6j2KueasPHXU3qLu2FKOls
+XbPy1XMuLYZgv8Zprcbs4KhQ3/A7/RO1cakxWlRwta63mUIM2xLIMIgRSR+DSZ5d
+JaDNO6i49eIGQXRxDb9dxA2hoCcoTv7PJKyOpNb5vyxMXJGY7H5j1jEEcqEeuI5u
+vuUwugQGtsl1eFLXIeQLerOHEQoS6wMv0fHBtZOVCHu8CCrnt/ag7kn39nkwNofL
+ovECAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgKkMB0GA1UdJQQWMBQGCCsGAQUFBwMC
+BggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDgQHBAUxMjM0NTAPBgNV
+HREECDAGhwR/AAABMA0GCSqGSIb3DQEBDQUAA4ICAQBjeGIfFqXuwHiClMytJNZL
+cRyjeZ6PJIAQtqh8Vi+XD2JiDTkwJ/g4R0FbgqE/icGkm/hsJ6BEwp8ep5eXevjS
+Hb8tVbM5Uc31yyIKcJMgnfS8O0eIXi5PxgFWPcUXxrsjwHyQREqj96HImmzOm99O
+MJhifWT3YP8OEMyl1KpioPaXafhc4ATEiRVZizHM9z+phyINBNghH3OaN91ZnsKJ
+El7mvOLjRi7fuSxBWJntKVAZAwXK+nH+z/Ay4AZFA9HgFHo3PGpKUaLOYCIsGxAq
+GP4V/WsOtEJ9rP5TR92pOvcj49T47FmwSYaRtoXHDVuoun0fdwT4DxWJdksqdWzG
+ieRls2IrZIvR2FT/A/XdQG3kZ79WA/K3OAGDgxv0PCpw6ssAMvgjR03TjEXpwMmN
+SNcrx1H6l8DHFHJN9f7SofO/J0hkA+fRZUFxP5R+P2BPU0hV14H9iSie/bxhSWIW
+ieAh0K1SNRbffXeYUvAgrjEvG5x40TktnvjHb20lxc1F1gqB+855kfZdiJeUeizi
+syq6OnCEp+RSBdK7J3scm7t6Nt3GRndJMO9hNDprogTqHxQbZ0jficntGd7Lbp+C
+CBegkhOzD6cp2rGlyYI+MmvdXFaHbsUJj2tfjHQdo2YjQ1s8r2pw219LTzPvO/Dz
+morZ618ezCBBqxHsDF6DCA==
+-----END CERTIFICATE-----

libbeat/common/transport/tlscommon/config.go

@@ -0,0 +1,86 @@
+package tlscommon
+
+import (
+	"crypto/tls"
+
+	"github.com/joeshaw/multierror"
+)
+
+// Config defines the user configurable options in the yaml file.
+type Config struct {
+	Enabled          *bool                   `config:"enabled"`
+	VerificationMode TLSVerificationMode     `config:"verification_mode"` // one of 'none', 'full'
+	Versions         []TLSVersion            `config:"supported_protocols"`
+	CipherSuites     []tlsCipherSuite        `config:"cipher_suites"`
+	CAs              []string                `config:"certificate_authorities"`
+	Certificate      CertificateConfig       `config:",inline"`
+	CurveTypes       []tlsCurveType          `config:"curve_types"`
+	Renegotiation    tlsRenegotiationSupport `config:"renegotiation"`
+}
+
+// LoadTLSConfig will load a certificate from config with all TLS based keys
+// defined. If Certificate and CertificateKey are configured, client authentication
+// will be configured. If no CAs are configured, the host CA will be used by go
+// built-in TLS support.
+func LoadTLSConfig(config *Config) (*TLSConfig, error) {
+	if !config.IsEnabled() {
+		return nil, nil
+	}
+
+	fail := multierror.Errors{}
+	logFail := func(es ...error) {
+		for _, e := range es {
+			if e != nil {
+				fail = append(fail, e)
+			}
+		}
+	}
+
+	var cipherSuites []uint16
+	for _, suite := range config.CipherSuites {
+		cipherSuites = append(cipherSuites, uint16(suite))
+	}
+
+	var curves []tls.CurveID
+	for _, id := range config.CurveTypes {
+		curves = append(curves, tls.CurveID(id))
+	}
+
+	cert, err := LoadCertificate(&config.Certificate)
+	logFail(err)
+
+	cas, errs := LoadCertificateAuthorities(config.CAs)
+	logFail(errs...)
+
+	// fail, if any error occurred when loading certificate files
+	if err = fail.Err(); err != nil {
+		return nil, err
+	}
+
+	var certs []tls.Certificate
+	if cert != nil {
+		certs = []tls.Certificate{*cert}
+	}
+
+	// return config if no error occurred
+	return &TLSConfig{
+		Versions:         config.Versions,
+		Verification:     config.VerificationMode,
+		Certificates:     certs,
+		RootCAs:          cas,
+		CipherSuites:     cipherSuites,
+		CurvePreferences: curves,
+		Renegotiation:    tls.RenegotiationSupport(config.Renegotiation),
+	}, nil
+}
+
+// Validate valies the TLSConfig struct making sure certificate sure we have both a certificate and
+// a key.
+func (c *Config) Validate() error {
+	return c.Certificate.Validate()
+}
+
+// IsEnabled returns true if the `enable` field is set to true in the yaml.
+func (c *Config) IsEnabled() bool {
+	return c != nil && (c.Enabled == nil || *c.Enabled)
+}

libbeat/common/transport/tlscommon/server_config.go

@@ -0,0 +1,84 @@
+package tlscommon
+
+import (
+	"crypto/tls"
+
+	"github.com/joeshaw/multierror"
+)
+
+// ServerConfig defines the user configurable tls options for any TCP based service.
+type ServerConfig struct {
+	Enabled          *bool               `config:"enabled"`
+	VerificationMode TLSVerificationMode `config:"verification_mode"` // one of 'none', 'full'
+	Versions         []TLSVersion        `config:"supported_protocols"`
+	CipherSuites     []tlsCipherSuite    `config:"cipher_suites"`
+	CAs              []string            `config:"certificate_authorities"`
+	Certificate      CertificateConfig   `config:",inline"`
+	CurveTypes       []tlsCurveType      `config:"curve_types"`
+	ClientAuth       tlsClientAuth       `config:"client_authentication"` //`none`, `optional` or `required`
+}
+
+// LoadTLSServerConfig tranforms a ServerConfig into a `tls.Config` to be used directly with golang
+// network types.
+func LoadTLSServerConfig(config *ServerConfig) (*TLSConfig, error) {
+	if !config.IsEnabled() {
+		return nil, nil
+	}
+
+	fail := multierror.Errors{}
+	logFail := func(es ...error) {
+		for _, e := range es {
+			if e != nil {
+				fail = append(fail, e)
+			}
+		}
+	}
+
+	var cipherSuites []uint16
+	for _, suite := range config.CipherSuites {
+		cipherSuites = append(cipherSuites, uint16(suite))
+	}
+
+	var curves []tls.CurveID
+	for _, id := range config.CurveTypes {
+		curves = append(curves, tls.CurveID(id))
+	}
+
+	cert, err := LoadCertificate(&config.Certificate)
+	logFail(err)
+
+	cas, errs := LoadCertificateAuthorities(config.CAs)
+	logFail(errs...)
+
+	// fail, if any error occurred when loading certificate files
+	if err = fail.Err(); err != nil {
+		return nil, err
+	}
+
+	var certs []tls.Certificate
+	if cert != nil {
+		certs = []tls.Certificate{*cert}
+	}
+
+	// return config if no error occurred
+	return &TLSConfig{
+		Versions:         config.Versions,
+		Verification:     config.VerificationMode,
+		Certificates:     certs,
+		ClientCAs:        cas,
+		CipherSuites:     cipherSuites,
+		CurvePreferences: curves,
+		ClientAuth:       tls.ClientAuthType(config.ClientAuth),
+	}, nil
+}
+
+// Validate valies the TLSConfig struct making sure certificate sure we have both a certificate and
+// a key.
+func (c *ServerConfig) Validate() error {
+	return c.Certificate.Validate()
+}
+
+// IsEnabled returns true if the `enable` field is set to true in the yaml.
+func (c *ServerConfig) IsEnabled() bool {
+	return c != nil && (c.Enabled == nil || *c.Enabled)
+}