Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Introduce alias support for index pattern #10075

Merged
merged 5 commits into from
Jan 21, 2019
Merged

Introduce alias support for index pattern #10075

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
bcc9877
Introduce alias support for index pattern
ruflin Jan 15, 2019
00f3eea
push partial fix
ruflin Jan 15, 2019
598d1ca
Copy alias info to `fieldFormatMap` too
Jan 16, 2019
e170ba6
add changelog
ruflin Jan 18, 2019
0403c04
make hound happy
ruflin Jan 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add `name` config option to `add_host_metadata` processor. {pull}9943[9943]
- Add `add_labels` and `add_tags` processors. {pull}9973[9973]
- Add missing file encoding to readers. {pull}10080[10080]
- Add alias field support in Kibana index pattern. {pull}10075[10075]

*Auditbeat*

Expand Down
33 changes: 33 additions & 0 deletions libbeat/common/field.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,13 @@ func (f Fields) HasKey(key string) bool {
return f.hasKey(keys)
}

// GetField returns the field in case it exists
func (f Fields) GetField(key string) *Field {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exported method Fields.GetField should have comment or be unexported

keys := strings.Split(key, ".")
return f.getField(keys)

}

// HasNode checks if inside fields the given node exists
// In contrast to HasKey it not only compares the leaf nodes but
// every single key it traverses.
Expand Down Expand Up @@ -216,6 +223,32 @@ func (f Fields) hasKey(keys []string) bool {
return false
}

func (f Fields) getField(keys []string) *Field {
// Nothing to compare anymore
if len(keys) == 0 {
return nil
}

key := keys[0]
keys = keys[1:]

for _, field := range f {
if field.Name == key {

if len(field.Fields) > 0 {
return field.Fields.getField(keys)
}
// Last entry in the tree but still more keys
if len(keys) > 0 {
return nil
}

return &field
}
}
return nil
}

// GetKeys returns a flat list of keys this Fields contains
func (f Fields) GetKeys() []string {
return f.getKeys("")
Expand Down
52 changes: 52 additions & 0 deletions libbeat/common/field_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,58 @@ func TestGetKeys(t *testing.T) {
}
}

func TestGetField(t *testing.T) {
tests := []struct {
fields Fields
key string
field *Field
}{
{
fields: Fields{
Field{
Name: "test", Fields: Fields{
Field{
Name: "find",
},
},
},
},
key: "test.find",
field: &Field{Name: "find"},
},
{
fields: Fields{
Field{
Name: "test", Fields: Fields{
Field{
Name: "find",
},
},
},
},
key: "test",
field: nil,
},
{
fields: Fields{
Field{
Name: "test", Fields: Fields{
Field{
Name: "find",
},
},
},
},
key: "test.find.more",
field: nil,
},
}

for _, test := range tests {
assert.Equal(t, test.field, test.fields.GetField(test.key))
}
}

func TestFieldValidate(t *testing.T) {
tests := []struct {
cfg MapStr
Expand Down
17 changes: 16 additions & 1 deletion libbeat/kibana/fields_transformer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"github.com/elastic/beats/libbeat/common"
)

var v640 = common.MustNewVersion("6.4.0")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Almost forgot that part.


type fieldsTransformer struct {
fields common.Fields
transformedFields []common.MapStr
Expand Down Expand Up @@ -84,6 +86,19 @@ func (t *fieldsTransformer) transformFields(commonFields common.Fields, path str
t.transformFields(f.Fields, f.Path)
}
} else {
if f.Type == "alias" {
if t.version.LessThan(v640) {
continue
}
if ff := t.fields.GetField(f.AliasPath); ff != nil {
// copy the field, keep
path := f.Path
name := f.Name
f = *ff
f.Path = path
f.Name = name
}
}
t.add(f)

if f.MultiFields != nil {
Expand All @@ -104,7 +119,7 @@ func (t *fieldsTransformer) update(target *common.MapStr, override common.Field)
target.Update(field)
if !override.Overwrite {
// compatible duplication
return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true'", override.Path)
return fmt.Errorf("field <%s> is duplicated, remove it or set 'overwrite: true', %+v, %+v", override.Path, override, field)
}
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions libbeat/kibana/index_pattern_generator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ func TestGenerate(t *testing.T) {
tmpDir := tmpPath(t)
defer os.RemoveAll(tmpDir)

v6, _ := common.NewVersion("6.0.0")
v6, _ := common.NewVersion("6.4.0")
versions := []*common.Version{v6}
for _, version := range versions {
generator, err := NewGenerator("beat-*", "b eat ?!", fieldsYml, tmpDir, "7.0.0-alpha1", *version)
Expand All @@ -149,7 +149,7 @@ func TestGenerateExtensive(t *testing.T) {
tmpDir := tmpPath(t)
defer os.RemoveAll(tmpDir)

version6, _ := common.NewVersion("6.0.0")
version6, _ := common.NewVersion("6.4.0")
versions := []*common.Version{version6}
for _, version := range versions {
generator, err := NewGenerator("metricbeat-*", "metric be at ?!", "testdata/extensive/fields.yml", tmpDir, "7.0.0-alpha1", *version)
Expand Down
4 changes: 2 additions & 2 deletions libbeat/kibana/testdata/beat-6.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
"objects": [
{
"attributes": {
"fieldFormatMap": "{\"long\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}}}",
"fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"blob\",\"scripted\":false,\"searchable\":false,\"type\":\"binary\"}]",
"fieldFormatMap": "{\"long\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}},\"alias\":{\"id\":\"url\",\"params\":{\"inputFormat\":\"string\",\"labelTemplate\":\"long template\",\"outputFormat\":\"float\",\"outputPrecision\":5,\"urlTemplate\":\"_a=(query:(language:lucene,query:'context.app.name:\\\"{{value}}\\\"'))\"}}}",
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@simitt Is there a good way to generate these files? I'm fighting to get the tests passing.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@simitt Is there a good way to generate these files? I'm fighting to get the tests passing.

Copy link
Contributor

@simitt simitt Jan 15, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ruflin you were asking how you could make the alias show up in the fieldFormat section of the Kibana index pattern. This is the entry point for handling them.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the way I make it show up fieldsFormat: https://github.com/elastic/beats/pull/10075/files#diff-086ab841dd8317d2aeec5107211984c2R87 Running it against Kibana this trick seems to work. My problem is with the tests and fieldFormatMap

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually I don't recall a good way for creating the tests automatically.

"fields": "[{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"long\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"multifield_field.keyword\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_id\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_type\",\"scripted\":false,\"searchable\":true,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_index\",\"scripted\":false,\"searchable\":false,\"type\":\"string\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"_score\",\"scripted\":false,\"searchable\":false,\"type\":\"number\"},{\"aggregatable\":false,\"analyzed\":false,\"count\":0,\"doc_values\":false,\"indexed\":false,\"name\":\"blob\",\"scripted\":false,\"searchable\":false,\"type\":\"binary\"},{\"aggregatable\":true,\"analyzed\":false,\"count\":0,\"doc_values\":true,\"indexed\":true,\"name\":\"alias\",\"scripted\":false,\"searchable\":true,\"type\":\"number\"}]",
"timeFieldName": "@timestamp",
"title": "beat-*",
"sourceFilters": "[{\"value\":\"user.email\"}, {\"value\": \"user.name\"},{\"value\": \"url.*\"}]"
Expand Down
6 changes: 5 additions & 1 deletion libbeat/kibana/testdata/fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,11 @@
- min_version: 6.0.0
value: "_a=(query:(language:lucene,query:'context.app.name:\"{{value}}\"'))"

- key: with source filter
- name: alias
type: alias
path: long

- key: with source filter
title: Test
kibana:
source_filters:
Expand Down