Add support for setting fields in the config file

CHANGELOG.asciidoc

@@ -187,6 +187,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 - Add builder support for autodiscover and annotations builder {pull}6408[6408]
 - Add plugin support for autodiscover builders, providers {pull}6457[6457]
 - Preserve runtime from container statuses in Kubernetes autodiscover {pull}6456[6456]
+- Experimental feature setup.template.append_fields added. {pull}6024[6024]
 
 *Auditbeat*
 

libbeat/common/field.go

@@ -99,6 +99,44 @@ func (f Fields) HasKey(key string) bool {
 	return f.hasKey(keys)
 }
 
+// HasNode checks if inside fields the given node exists
+// In contrast to HasKey it not only compares the leaf nodes but
+// every single key it traverses.
+func (f Fields) HasNode(key string) bool {
+	keys := strings.Split(key, ".")
+	return f.hasNode(keys)
+}
+
+func (f Fields) hasNode(keys []string) bool {
+
+	// Nothing to compare, so does not contain it
+	if len(keys) == 0 {
+		return false
+	}
+
+	key := keys[0]
+	keys = keys[1:]
+
+	for _, field := range f {
+
+		if field.Name == key {
+
+			//// It's the last key to compare
+			if len(keys) == 0 {
+				return true
+			}
+
+			// It's the last field to compare
+			if len(field.Fields) == 0 {
+				return true
+			}
+
+			return field.Fields.hasNode(keys)
+		}
+	}
+	return false
+}
+
 // Recursively generates the correct key based on the dots
 // The mapping requires "properties" between each layer. This is added here.
 func GenerateKey(key string) string {
@@ -134,3 +172,27 @@ func (f Fields) hasKey(keys []string) bool {
 	}
 	return false
 }
+
+// GetKeys returns a flat list of keys this Fields contains
+func (f Fields) GetKeys() []string {
+	return f.getKeys("")
+}
+
+func (f Fields) getKeys(namespace string) []string {
+
+	var keys []string
+
+	for _, field := range f {
+		fieldName := namespace + "." + field.Name
+		if namespace == "" {
+			fieldName = field.Name
+		}
+		if len(field.Fields) == 0 {
+			keys = append(keys, fieldName)
+		} else {
+			keys = append(keys, field.Fields.getKeys(fieldName)...)
+		}
+	}
+
+	return keys
+}

libbeat/common/field_test.go

@@ -68,7 +68,7 @@ func TestDynamicYaml(t *testing.T) {
 	}{
 		{
 			input: []byte(`
-name: test 
+name: test
 dynamic: true`),
 			output: Field{
 				Name:    "test",
@@ -115,3 +115,60 @@ dynamic: "strict"`),
 		}
 	}
 }
+
+func TestGetKeys(t *testing.T) {
+	tests := []struct {
+		fields Fields
+		keys   []string
+	}{
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			keys: []string{"test.find"},
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "a", Fields: Fields{
+						Field{
+							Name: "b",
+						},
+					},
+				},
+				Field{
+					Name: "a", Fields: Fields{
+						Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			keys: []string{"a.b", "a.c"},
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "a",
+				},
+				Field{
+					Name: "b",
+				},
+				Field{
+					Name: "c",
+				},
+			},
+			keys: []string{"a", "b", "c"},
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.keys, test.fields.GetKeys())
+	}
+}

libbeat/docs/template-config.asciidoc

@@ -82,3 +82,7 @@ setup.template.overwrite: false
 setup.template.settings:
   _source.enabled: false
 ----------------------------------------------------------------------
+
+*`setup.template.append_fields`*:: A list of of fields to be added to the template and Kibana index pattern. experimental[]
+
+NOTE: With append_fields only new fields can be added an no existing one overwritten or changed. This is especially useful if data is collected through the http/json metricset where the data structure is not known in advance. Changing the config of append_fields means the template has to be overwritten and only applies to new indices. If there are 2 Beats with different append_fields configs the last one writing the template will win. Any changes will also have an affect on the Kibana Index pattern.

libbeat/template/config.go

@@ -1,12 +1,15 @@
 package template
 
+import "github.com/elastic/beats/libbeat/common"
+
 type TemplateConfig struct {
-	Enabled   bool             `config:"enabled"`
-	Name      string           `config:"name"`
-	Pattern   string           `config:"pattern"`
-	Fields    string           `config:"fields"`
-	Overwrite bool             `config:"overwrite"`
-	Settings  TemplateSettings `config:"settings"`
+	Enabled      bool             `config:"enabled"`
+	Name         string           `config:"name"`
+	Pattern      string           `config:"pattern"`
+	Fields       string           `config:"fields"`
+	AppendFields common.Fields    `config:"append_fields"`
+	Overwrite    bool             `config:"overwrite"`
+	Settings     TemplateSettings `config:"settings"`
 }
 
 type TemplateSettings struct {

libbeat/template/template.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/elastic/beats/libbeat/beat"
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/common/fmtstr"
 )
 
@@ -24,7 +25,7 @@ type Template struct {
 	pattern     string
 	beatVersion common.Version
 	esVersion   common.Version
-	settings    TemplateSettings
+	config      TemplateConfig
 }
 
 // New creates a new template instance
@@ -87,17 +88,26 @@ func New(beatVersion string, beatName string, esVersion string, config TemplateC
 		name:        name,
 		beatVersion: *bV,
 		esVersion:   *esV,
-		settings:    config.Settings,
+		config:      config,
 	}, nil
 }
 
 // Load the given input and generates the input based on it
 func (t *Template) Load(file string) (common.MapStr, error) {
+
 	fields, err := common.LoadFieldsYaml(file)
 	if err != nil {
 		return nil, err
 	}
 
+	if len(t.config.AppendFields) > 0 {
+		cfgwarn.Experimental("append_fields is used.")
+		fields, err = appendFields(fields, t.config.AppendFields)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Start processing at the root
 	properties := common.MapStr{}
 	processor := Processor{EsVersion: t.esVersion}
@@ -155,7 +165,7 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 		indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards)
 	}
 
-	indexSettings.DeepUpdate(t.settings.Index)
+	indexSettings.DeepUpdate(t.config.Settings.Index)
 
 	var mappingName string
 	if t.esVersion.Major >= 6 {
@@ -182,9 +192,9 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 		},
 	}
 
-	if len(t.settings.Source) > 0 {
+	if len(t.config.Settings.Source) > 0 {
 		key := fmt.Sprintf("mappings.%s._source", mappingName)
-		basicStructure.Put(key, t.settings.Source)
+		basicStructure.Put(key, t.config.Settings.Source)
 	}
 
 	// ES 6 moved from template to index_patterns: https://github.com/elastic/elasticsearch/pull/21009
@@ -200,3 +210,19 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common.
 
 	return basicStructure
 }
+
+func appendFields(fields, appendFields common.Fields) (common.Fields, error) {
+	if len(appendFields) > 0 {
+		appendFieldKeys := appendFields.GetKeys()
+
+		// Append is only allowed to add fields, not overwrite
+		for _, key := range appendFieldKeys {
+			if fields.HasNode(key) {
+				return nil, fmt.Errorf("append_fields contains an already existing key: %s", key)
+			}
+		}
+		// Appends fields to existing fields
+		fields = append(fields, appendFields...)
+	}
+	return fields, nil
+}

libbeat/template/template_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/libbeat/common"
 )
 
 func TestNumberOfRoutingShards(t *testing.T) {
@@ -54,3 +56,106 @@ func TestNumberOfRoutingShardsOverwrite(t *testing.T) {
 
 	assert.Equal(t, 5, shards.(int))
 }
+
+func TestAppendFields(t *testing.T) {
+	tests := []struct {
+		fields       common.Fields
+		appendFields common.Fields
+		error        bool
+	}{
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "b",
+						},
+					},
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			error: false,
+		},
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "b",
+						},
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			error: true,
+		},
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			error: true,
+		},
+		{
+			fields: common.Fields{
+				common.Field{
+					Name: "a",
+					Fields: common.Fields{
+						common.Field{
+							Name: "c",
+						},
+					},
+				},
+			},
+			appendFields: common.Fields{
+				common.Field{
+					Name: "a",
+				},
+			},
+			error: true,
+		},
+	}
+
+	for _, test := range tests {
+		_, err := appendFields(test.fields, test.appendFields)
+		if test.error {
+			assert.Error(t, err)
+		} else {
+			assert.NoError(t, err)
+		}
+	}
+}