Cherry-pick #18127 to 7.x: Add Kerberos-aware Elasticsearch and integ…

…ration test to ES output (#18168)

* Add Kerberos-aware Elasticsearch and integration test to ES output (#18127)

This PR adds an integration test to the Elasticsearch output to check Kerberos authentication.

Furthermore, it adds a new element to our testing environment, a Kerberos-aware Elasticsearch instance named `elasticsearch_kerberos.elastic`.
(cherry picked from commit 428ee72)

* downgrade dockerfile version to 7.7

libbeat/docker-compose.yml

@@ -27,6 +27,7 @@ services:
       - ES_MONITORING_HOST=elasticsearch_monitoring
       - ES_MONITORING_PORT=9200
       - ES_HOST_SSL=elasticsearchssl
+      - ES_KERBEROS_HOST=elasticsearch_kerberos.elastic
       - ES_PORT_SSL=9200
       - ES_SUPERUSER_USER=admin
       - ES_SUPERUSER_PASS=changeme
@@ -42,14 +43,15 @@ services:
   proxy_dep:
     image: busybox
     depends_on:
-      elasticsearch:            { condition: service_healthy }
-      elasticsearch_monitoring: { condition: service_healthy }
-      elasticsearchssl:         { condition: service_healthy }
-      logstash:                 { condition: service_healthy }
-      kafka:                    { condition: service_healthy }
-      redis:                    { condition: service_healthy }
-      sredis:                   { condition: service_healthy }
-      kibana:                   { condition: service_healthy }
+      elasticsearch:                  { condition: service_healthy }
+      elasticsearch_kerberos.elastic: { condition: service_healthy }
+      elasticsearch_monitoring:       { condition: service_healthy }
+      elasticsearchssl:               { condition: service_healthy }
+      logstash:                       { condition: service_healthy }
+      kafka:                          { condition: service_healthy }
+      redis:                          { condition: service_healthy }
+      sredis:                         { condition: service_healthy }
+      kibana:                         { condition: service_healthy }
     healthcheck:
       interval: 1s
       retries: 1200
@@ -128,6 +130,35 @@ services:
     environment:
       - ADVERTISED_HOST=kafka
 
+  elasticsearch_kerberos.elastic:
+    build: ${ES_BEATS}/testing/environments/docker/elasticsearch_kerberos
+    healthcheck:
+      test: bash -c "/healthcheck.sh"
+      retries: 1200
+      interval: 5s
+      start_period: 60s
+    environment:
+      - "TERM=linux"
+      - "ELASTIC_PASSWORD=changeme"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Djava.security.krb5.conf=/etc/krb5.conf"
+      - "network.host="
+      - "transport.host=127.0.0.1"
+      - "http.host=0.0.0.0"
+      - "xpack.security.enabled=true"
+      - "indices.id_field_data.enabled=true"
+      - "xpack.license.self_generated.type=trial"
+      - "xpack.security.authc.realms.kerberos.ELASTIC.order=1"
+      - "xpack.security.authc.realms.kerberos.ELASTIC.keytab.path=/usr/share/elasticsearch/config/HTTP_elasticsearch_kerberos.elastic.keytab"
+    hostname: elasticsearch_kerberos.elastic
+    volumes:
+      # This is needed otherwise there won't be enough entropy to generate a new kerberos realm
+      - /dev/urandom:/dev/random
+    ports:
+      - 1088
+      - 1749
+      - 9200
+    command: bash -c "/start.sh"
+
   kibana:
     extends:
       file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml

libbeat/esleg/eslegtest/util.go

@@ -64,6 +64,11 @@ func GetEsHost() string {
 	return getEnv("ES_HOST", ElasticsearchDefaultHost)
 }
 
+// GetEsKerberosHost returns the Elasticsearch testing host.
+func GetEsKerberosHost() string {
+	return getEnv("ES_KERBEROS_HOST", ElasticsearchDefaultHost)
+}
+
 // getEsPort returns the Elasticsearch testing port.
 func getEsPort() string {
 	return getEnv("ES_PORT", ElasticsearchDefaultPort)

libbeat/outputs/elasticsearch/client_integration_test.go

@@ -21,6 +21,7 @@ package elasticsearch
 
 import (
 	"context"
+	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
@@ -43,9 +44,39 @@ import (
 
 func TestClientPublishEvent(t *testing.T) {
 	index := "beat-int-pub-single-event"
-	output, client := connectTestEs(t, map[string]interface{}{
+	cfg := map[string]interface{}{
 		"index": index,
-	})
+	}
+
+	testPublishEvent(t, index, cfg)
+}
+
+func TestClientPublishEventKerberosAware(t *testing.T) {
+	err := setupRoleMapping(t, eslegtest.GetEsKerberosHost())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	index := "beat-int-pub-single-event-behind-kerb"
+	cfg := map[string]interface{}{
+		"hosts":    eslegtest.GetEsKerberosHost(),
+		"index":    index,
+		"username": "",
+		"password": "",
+		"kerberos": map[string]interface{}{
+			"auth_type":   "password",
+			"config_path": "testdata/krb5.conf",
+			"username":    eslegtest.GetUser(),
+			"password":    eslegtest.GetPass(),
+			"realm":       "ELASTIC",
+		},
+	}
+
+	testPublishEvent(t, index, cfg)
+}
+
+func testPublishEvent(t *testing.T, index string, cfg map[string]interface{}) {
+	output, client := connectTestEs(t, cfg)
 
 	// drop old index preparing test
 	client.conn.Delete(index, "", "", nil)
@@ -281,6 +312,33 @@ func connectTestEs(t *testing.T, cfg interface{}) (outputs.Client, *Client) {
 	return client, client
 }
 
+// setupRoleMapping sets up role mapping for the Kerberos user beats@ELASTIC
+func setupRoleMapping(t *testing.T, host string) error {
+	_, client := connectTestEs(t, map[string]interface{}{
+		"hosts":    host,
+		"username": "elastic",
+		"password": "changeme",
+	})
+
+	roleMappingURL := client.conn.URL + "/_security/role_mapping/kerbrolemapping"
+
+	status, _, err := client.conn.RequestURL("POST", roleMappingURL, map[string]interface{}{
+		"roles":   []string{"superuser"},
+		"enabled": true,
+		"rules": map[string]interface{}{
+			"field": map[string]interface{}{
+				"username": "beats@ELASTIC",
+			},
+		},
+	})
+
+	if status >= 300 {
+		return fmt.Errorf("non-2xx return code: %d", status)
+	}
+
+	return err
+}
+
 func randomClient(grp outputs.Group) outputs.NetworkClient {
 	L := len(grp.Clients)
 	if L == 0 {

libbeat/outputs/elasticsearch/testdata/krb5.conf

@@ -0,0 +1,43 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[libdefaults]
+    default_realm = ELASTIC
+    dns_canonicalize_hostname = false
+    dns_lookup_kdc = false
+    dns_lookup_realm = false
+    dns_uri_lookup = false
+    forwardable = true
+    ignore_acceptor_hostname = true
+    rdns = false
+    default_tgs_enctypes = aes128-cts-hmac-sha1-96
+    default_tkt_enctypes = aes128-cts-hmac-sha1-96
+    permitted_enctypes = aes128-cts-hmac-sha1-96
+    udp_preference_limit = 1
+    kdc_timeout = 3000
+
+[realms]
+    ELASTIC = {
+        kdc = elasticsearch_kerberos.elastic:1088
+        admin_server = elasticsearch_kerberos.elastic:1749
+        default_domain = elastic
+    }
+
+[domain_realm]
+    .elastic = ELASTIC
+    elastic = ELASTIC
+

testing/environments/docker/elasticsearch/kerberos/init.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# setup Keberos
+echo elasticsearch_kerberos.elastic > /etc/hostname && echo "127.0.0.1 elasticsearch_kerberos.elastic" >> /etc/hosts
+
+/scripts/installkdc.sh
+/scripts/addprincs.sh
+
+# add test user
+bin/elasticsearch-users useradd beats -r superuser -p testing | /usr/local/bin/docker-entrypoint.sh eswrapper

testing/environments/docker/elasticsearch/kerberos/installkdc.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# KDC installation steps and considerations based on https://web.mit.edu/kerberos/krb5-latest/doc/admin/install_kdc.html
+# and helpful input from https://help.ubuntu.com/community/Kerberos
+
+LOCALSTATEDIR=/etc
+LOGDIR=/var/log/krb5
+
+#MARKER_FILE=/etc/marker
+
+# Transfer and interpolate krb5.conf
+cp /config/krb5.conf.template $LOCALSTATEDIR/krb5.conf
+sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5.conf
+sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5.conf
+sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5.conf
+sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf
+
+
+# Transfer and interpolate the kdc.conf
+mkdir -p $LOCALSTATEDIR/krb5kdc
+cp /config/kdc.conf.template $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${REALM_NAME}/'$REALM_NAME'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${KDC_NAME}/'$KDC_NAME'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${BUILD_ZONE}/'$BUILD_ZONE'/g' $LOCALSTATEDIR/krb5kdc/kdc.conf
+sed -i 's/${ELASTIC_ZONE}/'$ELASTIC_ZONE'/g' $LOCALSTATEDIR/krb5.conf
+
+# Touch logging locations
+mkdir -p $LOGDIR
+touch $LOGDIR/kadmin.log
+touch $LOGDIR/krb5kdc.log
+touch $LOGDIR/krb5lib.log
+
+# Update package manager
+yum update -qqy
+
+# Install krb5 packages
+yum install -qqy krb5-{server,libs,workstation}
+
+# Create kerberos database with stash file and garbage password
+kdb5_util create -s -r $REALM_NAME -P zyxwvutsrpqonmlk9876
+
+# Set up admin acls
+cat << EOF > /etc/krb5kdc/kadm5.acl
+*/admin@$REALM_NAME	*
+*@$REALM_NAME   	*
+*/*@$REALM_NAME		i
+EOF
+
+# Create admin principal
+kadmin.local -q "addprinc -pw elastic admin/admin@$REALM_NAME"
+kadmin.local -q "ktadd -k /etc/admin.keytab admin/admin@$REALM_NAME"
+
+# Create a link so addprinc.sh is on path
+ln -s /scripts/addprinc.sh /usr/bin/

testing/environments/docker/elasticsearch_kerberos/Dockerfile

@@ -0,0 +1,15 @@
+FROM docker.elastic.co/elasticsearch/elasticsearch:7.7.0-SNAPSHOT
+
+ADD scripts /scripts
+ADD config /config
+ADD healthcheck.sh /healthcheck.sh
+ADD start.sh /start.sh
+
+ENV REALM_NAME ELASTIC
+ENV KDC_NAME elasticsearch_kerberos.elastic
+ENV BUILD_ZONE elastic
+ENV ELASTIC_ZONE $BUILD_ZONE
+
+USER root
+RUN /scripts/installkdc.sh && /scripts/addprincs.sh
+USER elasticsearch

testing/environments/docker/elasticsearch_kerberos/config/kdc.conf.template

@@ -0,0 +1,34 @@
+# Licensed to Elasticsearch under one or more contributor
+# license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright
+# ownership. Elasticsearch licenses this file to you under
+# the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[kdcdefaults]
+    kdc_listen = 1088
+    kdc_tcp_listen = 1088
+
+[realms]
+    ${REALM_NAME} = {
+        kadmind_port = 1749
+        max_life = 12h 0m 0s
+        max_renewable_life = 7d 0h 0m 0s
+        master_key_type = aes256-cts
+        supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
+    }
+
+[logging]
+    kdc = FILE:/var/log/krb5/krb5kdc.log
+    admin_server = FILE:/var/log/krb5/kadmin.log
+    default = FILE:/var/log/krb5/krb5lib.log

testing/environments/docker/elasticsearch_kerberos/config/krb5.conf

@@ -0,0 +1,25 @@
+[libdefaults]
+    default_realm = ELASTIC
+    dns_canonicalize_hostname = false
+    dns_lookup_kdc = false
+    dns_lookup_realm = false
+    dns_uri_lookup = false
+    forwardable = true
+    ignore_acceptor_hostname = true
+    rdns = false
+    default_tgs_enctypes = aes128-cts-hmac-sha1-96
+    default_tkt_enctypes = aes128-cts-hmac-sha1-96
+    permitted_enctypes = aes128-cts-hmac-sha1-96
+    kdc_timeout = 3000
+
+[realms]
+    ELASTIC = {
+        kdc = elasticsearch_kerberos.elastic:88
+        admin_server = elasticsearch_kerberos.elastic:749
+        default_domain = elastic
+    }
+
+[domain_realm]
+    .elastic = ELASTIC
+    elastic = ELASTIC
+