From 1edbd779b041707c42599778071f18b755e78211 Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 14 Nov 2017 11:01:04 +1100 Subject: [PATCH] Add number_of_routing_shards config set to 30 With https://github.com/elastic/elasticsearch/pull/26931 the possibility for splitting shards was introduced. To make use of this feature for indices created with ES >=6.1 the config option `index.number_of_routing_shards` is required. This adds this config option currently set to 30 as it's a multiple of 1, 3 and 5, our current number of default shards in Beats and ES. This allows users with default configs to scale their split their shards. The `number_of_routing_shards` can also be overwritten in the config file. --- CHANGELOG.asciidoc | 1 + auditbeat/auditbeat.reference.yml | 1 + filebeat/filebeat.reference.yml | 1 + heartbeat/heartbeat.reference.yml | 1 + libbeat/_meta/config.reference.yml | 1 + libbeat/template/template.go | 12 +++++-- libbeat/template/template_test.go | 56 +++++++++++++++++++++++++++++ metricbeat/metricbeat.reference.yml | 1 + packetbeat/packetbeat.reference.yml | 1 + winlogbeat/winlogbeat.reference.yml | 1 + 10 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 libbeat/template/template_test.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1e3159c7644..813df6f2ce4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -94,6 +94,7 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di - Refactor add_kubernetes_metadata to support autodiscovery {pull}5434[5434] - Improve custom flag handling and CLI flags usage message. {pull}5543[5543] - Update to Golang 1.9.2 +- Add number_of_routing_shards config set to 30 {pull}5570[5570] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index 762f80a892b..dfad07b2638 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -755,6 +755,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 4c146016c57..293d834a79c 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1177,6 +1177,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index 248f10c9aad..5551500da64 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -875,6 +875,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index a7d1196d052..e1bebc42897 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -661,6 +661,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/libbeat/template/template.go b/libbeat/template/template.go index 81b4148ef4d..31420d5f10c 100644 --- a/libbeat/template/template.go +++ b/libbeat/template/template.go @@ -11,8 +11,9 @@ import ( var ( // Defaults used in the template - defaultDateDetection = false - defaultTotalFieldsLimit = 10000 + defaultDateDetection = false + defaultTotalFieldsLimit = 10000 + defaultNumberOfRoutingShards = 30 // Array to store dynamicTemplate parts in dynamicTemplates []common.MapStr @@ -147,6 +148,13 @@ func (t *Template) generate(properties common.MapStr, dynamicTemplates []common. }, }, } + + // number_of_routing shards is only supported for ES version >= 6.1 + version61, _ := common.NewVersion("6.1.0") + if !t.esVersion.LessThan(version61) { + indexSettings.Put("number_of_routing_shards", defaultNumberOfRoutingShards) + } + indexSettings.DeepUpdate(t.settings.Index) var mappingName string diff --git a/libbeat/template/template_test.go b/libbeat/template/template_test.go new file mode 100644 index 00000000000..2e5200f73ad --- /dev/null +++ b/libbeat/template/template_test.go @@ -0,0 +1,56 @@ +// +build !integration + +package template + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNumberOfRoutingShards(t *testing.T) { + + beatVersion := "6.1.0" + beatName := "testbeat" + config := TemplateConfig{} + + // Test it exists in 6.1 + template, err := New(beatVersion, beatName, "6.1.0", config) + assert.NoError(t, err) + + data := template.generate(nil, nil) + shards, err := data.GetValue("settings.index.number_of_routing_shards") + assert.NoError(t, err) + + assert.Equal(t, 30, shards.(int)) + + // Test it does not exist in 6.0 + template, err = New(beatVersion, beatName, "6.0.0", config) + assert.NoError(t, err) + + data = template.generate(nil, nil) + shards, err = data.GetValue("settings.index.number_of_routing_shards") + assert.Error(t, err) + assert.Equal(t, nil, shards) +} + +func TestNumberOfRoutingShardsOverwrite(t *testing.T) { + + beatVersion := "6.1.0" + beatName := "testbeat" + config := TemplateConfig{ + Settings: TemplateSettings{ + Index: map[string]interface{}{"number_of_routing_shards": 5}, + }, + } + + // Test it exists in 6.1 + template, err := New(beatVersion, beatName, "6.1.0", config) + assert.NoError(t, err) + + data := template.generate(nil, nil) + shards, err := data.GetValue("settings.index.number_of_routing_shards") + assert.NoError(t, err) + + assert.Equal(t, 5, shards.(int)) +} diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index f5d8b91a514..e2dc16cb07a 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -1142,6 +1142,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 9d93f1fe19e..237c72291db 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -1125,6 +1125,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index 7382c9eba4d..1ff92640cc8 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -690,6 +690,7 @@ setup.template.settings: #index: #number_of_shards: 1 #codec: best_compression + #number_of_routing_shards: 30 # A dictionary of settings for the _source field. For more details, please check # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html