Introduce alias support for index pattern (#10075)

The goal of this PR is that in case an alias is detected in the fields.yml, for the index pattern the fields from the original are copied over.

CHANGELOG.next.asciidoc

@@ -129,6 +129,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `name` config option to `add_host_metadata` processor. {pull}9943[9943]
 - Add `add_labels` and `add_tags` processors. {pull}9973[9973]
 - Add missing file encoding to readers. {pull}10080[10080]
+- Add alias field support in Kibana index pattern. {pull}10075[10075]
 
 *Auditbeat*
 

libbeat/common/field.go

@@ -142,6 +142,13 @@ func (f Fields) HasKey(key string) bool {
 	return f.hasKey(keys)
 }
 
+// GetField returns the field in case it exists
+func (f Fields) GetField(key string) *Field {
+	keys := strings.Split(key, ".")
+	return f.getField(keys)
+
+}
+
 // HasNode checks if inside fields the given node exists
 // In contrast to HasKey it not only compares the leaf nodes but
 // every single key it traverses.
@@ -216,6 +223,32 @@ func (f Fields) hasKey(keys []string) bool {
 	return false
 }
 
+func (f Fields) getField(keys []string) *Field {
+	// Nothing to compare anymore
+	if len(keys) == 0 {
+		return nil
+	}
+
+	key := keys[0]
+	keys = keys[1:]
+
+	for _, field := range f {
+		if field.Name == key {
+
+			if len(field.Fields) > 0 {
+				return field.Fields.getField(keys)
+			}
+			// Last entry in the tree but still more keys
+			if len(keys) > 0 {
+				return nil
+			}
+
+			return &field
+		}
+	}
+	return nil
+}
+
 // GetKeys returns a flat list of keys this Fields contains
 func (f Fields) GetKeys() []string {
 	return f.getKeys("")

libbeat/common/field_test.go

@@ -192,6 +192,58 @@ func TestGetKeys(t *testing.T) {
 	}
 }
 
+func TestGetField(t *testing.T) {
+	tests := []struct {
+		fields Fields
+		key    string
+		field  *Field
+	}{
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			key:   "test.find",
+			field: &Field{Name: "find"},
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			key:   "test",
+			field: nil,
+		},
+		{
+			fields: Fields{
+				Field{
+					Name: "test", Fields: Fields{
+						Field{
+							Name: "find",
+						},
+					},
+				},
+			},
+			key:   "test.find.more",
+			field: nil,
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.field, test.fields.GetField(test.key))
+	}
+}
+
 func TestFieldValidate(t *testing.T) {
 	tests := []struct {
 		cfg   MapStr

libbeat/kibana/fields_transformer.go

@@ -24,6 +24,8 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 )
 
+var v640 = common.MustNewVersion("6.4.0")
+
 type fieldsTransformer struct {
 	fields                    common.Fields
 	transformedFields         []common.MapStr
@@ -84,6 +86,19 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
 				t.transformFields(f.Fields, f.Path)
 			}
 		} else {
+			if f.Type == "alias" {
+				if t.version.LessThan(v640) {
+					continue
+				}
+				if ff := t.fields.GetField(f.AliasPath); ff != nil {
+					// copy the field, keep
+					path := f.Path
+					name := f.Name
+					f = *ff
+					f.Path = path
+					f.Name = name
+				}
+			}
 			t.add(f)
 
 			if f.MultiFields != nil {
@@ -104,7 +119,7 @@ func (t *fieldsTransformer) update(target *common.MapStr, override common.Field)
 		target.Update(field)
 		if !override.Overwrite {
 			// compatible duplication
-			return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true'", override.Path)
+			return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true', %+v, %+v", override.Path, override, field)
 		}
 		return nil
 	}

libbeat/kibana/index_pattern_generator_test.go

@@ -122,7 +122,7 @@ func TestGenerate(t *testing.T) {
 	tmpDir := tmpPath(t)
 	defer os.RemoveAll(tmpDir)
 
-	v6, _ := common.NewVersion("6.0.0")
+	v6, _ := common.NewVersion("6.4.0")
 	versions := []*common.Version{v6}
 	for _, version := range versions {
 		generator, err := NewGenerator("beat-*", "b eat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *version)
@@ -149,7 +149,7 @@ func TestGenerateExtensive(t *testing.T) {
 	tmpDir := tmpPath(t)
 	defer os.RemoveAll(tmpDir)
 
-	version6, _ := common.NewVersion("6.0.0")
+	version6, _ := common.NewVersion("6.4.0")
 	versions := []*common.Version{version6}
 	for _, version := range versions {
 		generator, err := NewGenerator("metricbeat-*", "metric be at ?!", "testdata/extensive/fields.yml", tmpDir, "7.0.0-alpha1", *version)

libbeat/kibana/testdata/beat-6.json

@@ -2,8 +2,8 @@
   "objects": [
     {
       "attributes": {
-        "fieldFormatMap": "{\"long\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}}}",
-        "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"blob\",\"scripted\":false,\"searchable\":false,\"type\":\"binary\"}]",
+        "fieldFormatMap": "{\"long\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}},\"alias\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}}}",
+        "fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"blob\",\"scripted\":false,\"searchable\":false,\"type\":\"binary\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"alias\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"}]",
         "timeFieldName": "@timestamp",
         "title": "beat-*",
         "sourceFilters": "[{\"value\":\"user.email\"}, {\"value\": \"user.name\"},{\"value\": \"url.*\"}]"

libbeat/kibana/testdata/fields.yml

@@ -25,7 +25,11 @@
         - min_version: 6.0.0
           value: "_a=(query:(language:lucene,query:'context.app.name:\"{{value}}\"'))"
 
-- key: with source filter 
+    - name: alias
+      type: alias
+      path: long
+
+- key: with source filter
   title: Test
   kibana:
     source_filters: