diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index d4bbd0d5eae..f8a465a02e2 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -38,4 +38,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067] - Update urllib3 version to 1.24.2 {pull}11930[11930] - Add libbeat/common/cleanup package. {pull}12134[12134] -- Only Load minimal template if no fields are provided. {pull}12103[12103] \ No newline at end of file +- Only Load minimal template if no fields are provided. {pull}12103[12103] +- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132] \ No newline at end of file diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 78ea2ab62fa..77436a6cfc9 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error { //SetupSettings holds settings necessary for beat setup type SetupSettings struct { - Template bool Dashboard bool MachineLearning bool Pipeline bool - ILMPolicy bool + IndexManagement bool + //Deprecated: use IndexManagementKey instead + Template bool + //Deprecated: use IndexManagementKey instead + ILMPolicy bool } // Setup registers ES index template, kibana dashboards, ml jobs and pipelines. @@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er return err } - if setup.Template || setup.ILMPolicy { + if setup.IndexManagement || setup.Template || setup.ILMPolicy { outCfg := b.Config.Output - if outCfg.Name() != "elasticsearch" { return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled") } + esClient, err := elasticsearch.NewConnectedClient(outCfg.Config()) + if err != nil { + return err + } - esConfig := outCfg.Config() - if b.IdxSupporter.Enabled() { - esClient, err := elasticsearch.NewConnectedClient(esConfig) - if err != nil { - return err - } - - // prepare index by loading templates, lifecycle policies and write aliases - - m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) - var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset - if setup.Template { - tmplLoadMode = idxmgmt.LoadModeOverwrite - } - if setup.ILMPolicy { - ilmLoadMode = idxmgmt.LoadModeOverwrite - } - - err = m.Setup(tmplLoadMode, ilmLoadMode) - if err != nil { - return err - } + var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset + if setup.IndexManagement || setup.Template { + loadTemplate = idxmgmt.LoadModeOverwrite + } + if setup.IndexManagement || setup.ILMPolicy { + loadILM = idxmgmt.LoadModeOverwrite + } + m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields)) + if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok { + fmt.Println(warn) + } + if err = m.Setup(loadTemplate, loadILM); err != nil { + return err } - fmt.Println("Index setup complete.") + fmt.Println("Index setup finished.") } if setup.Dashboard { diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index eeed4389e8f..f3ed31b7afb 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -28,15 +28,23 @@ import ( ) const ( - //TemplateKey used for defining template in setup cmd - TemplateKey = "template" //DashboardKey used for registering dashboards in setup cmd DashboardKey = "dashboards" //MachineLearningKey used for registering ml jobs in setup cmd MachineLearningKey = "machine-learning" //PipelineKey used for registering pipelines in setup cmd PipelineKey = "pipelines" - //ILMPolicyKey used for registering ilm in setup cmd + //IndexManagementKey used for loading all components related to ES index management in setup cmd + IndexManagementKey = "index-management" + + //TemplateKey used for loading template in setup cmd + // + //Deprecated: use IndexManagementKey instead + TemplateKey = "template" + + //ILMPolicyKey used for loading ilm in setup cmd + // + //Deprecated: use IndexManagementKey instead ILMPolicyKey = "ilm-policy" ) @@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co } var registeredFlags = map[string]bool{ - TemplateKey: false, DashboardKey: false, MachineLearningKey: false, PipelineKey: false, + IndexManagementKey: false, + TemplateKey: false, ILMPolicyKey: false, } var setupAll = true @@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co for k, v := range registeredFlags { if setupAll || v { switch k { - case TemplateKey: - s.Template = true case DashboardKey: s.Dashboard = true case MachineLearningKey: s.MachineLearning = true case PipelineKey: s.Pipeline = true + case IndexManagementKey: + s.IndexManagement = true case ILMPolicyKey: s.ILMPolicy = true + case TemplateKey: + s.Template = true } } } @@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co }, } - setup.Flags().Bool(TemplateKey, false, "Setup index template") setup.Flags().Bool(DashboardKey, false, "Setup dashboards") setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations") setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines") + setup.Flags().Bool(IndexManagementKey, false, + "Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias") + setup.Flags().Bool(TemplateKey, false, "Setup index template") + setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey)) setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy") + setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey)) return &setup } diff --git a/libbeat/idxmgmt/componenttype_string.go b/libbeat/idxmgmt/componenttype_string.go new file mode 100644 index 00000000000..5718922aac9 --- /dev/null +++ b/libbeat/idxmgmt/componenttype_string.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by "stringer -linecomment -type componentType"; DO NOT EDIT. + +package idxmgmt + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[componentTemplate-0] + _ = x[componentILM-1] +} + +const _componentType_name = "templateilm" + +var _componentType_index = [...]uint8{0, 8, 11} + +func (i componentType) String() string { + if i >= componentType(len(_componentType_index)-1) { + return "componentType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _componentType_name[_componentType_index[i]:_componentType_index[i+1]] +} diff --git a/libbeat/idxmgmt/idxmgmt.go b/libbeat/idxmgmt/idxmgmt.go index 6ec75b033ef..6837af03d13 100644 --- a/libbeat/idxmgmt/idxmgmt.go +++ b/libbeat/idxmgmt/idxmgmt.go @@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er // A manager instantiated via Supporter is responsible for instantiating/configuring // the index throughout the Elastic Stack. type Supporter interface { - // Enalbed checks if index management is configured to configure templates, - // ILM, or aliases. + // Enabled checks if index management is configured to setup templates or ILM Enabled() bool // BuildSelector create an index selector. @@ -62,6 +61,9 @@ type Asseter interface { // Manager is used to initialize indices, ILM policies, and aliases within the // Elastic Stack. type Manager interface { + VerifySetup(template, ilm LoadMode) (bool, string) + // When supporting index lifecycle management, ensure templates and policies + // are created before write aliases, to ensure templates are applied to the indices. Setup(template, ilm LoadMode) error } diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go index ea655bb71a8..13d1a201fcf 100644 --- a/libbeat/idxmgmt/std.go +++ b/libbeat/idxmgmt/std.go @@ -62,6 +62,33 @@ type ilmIndexSelector struct { st *indexState } +type componentType uint8 + +//go:generate stringer -linecomment -type componentType +const ( + componentTemplate componentType = iota //template + componentILM //ilm +) + +type feature struct { + component componentType + enabled, overwrite, load bool +} + +func newFeature(c componentType, enabled, overwrite bool, mode LoadMode) feature { + if mode == LoadModeUnset && !enabled { + mode = LoadModeDisabled + } + if mode >= LoadModeOverwrite { + overwrite = true + } + if mode == LoadModeForce { + enabled = true + } + load := mode.Enabled() && enabled + return feature{component: c, enabled: enabled, overwrite: overwrite, load: load} +} + func newIndexSupport( log *logp.Logger, info beat.Info, @@ -74,7 +101,7 @@ func newIndexSupport( ilmFactory = ilm.DefaultSupport } - ilm, err := ilmFactory(log, info, ilmConfig) + ilmSupporter, err := ilmFactory(log, info, ilmConfig) if err != nil { return nil, err } @@ -86,7 +113,7 @@ func newIndexSupport( return &indexSupport{ log: log, - ilm: ilm, + ilm: ilmSupporter, info: info, templateCfg: tmplCfg, migration: migration, @@ -95,17 +122,26 @@ func newIndexSupport( } func (s *indexSupport) Enabled() bool { - return s.templateCfg.Enabled || (s.ilm.Mode() != ilm.ModeDisabled) + return s.enabled(componentTemplate) || s.enabled(componentILM) +} + +func (s *indexSupport) enabled(c componentType) bool { + switch c { + case componentTemplate: + return s.templateCfg.Enabled + case componentILM: + return s.ilm.Mode() != ilm.ModeDisabled + } + return false } func (s *indexSupport) Manager( clientHandler ClientHandler, assets Asseter, ) Manager { - ilm := s.ilm.Manager(clientHandler) return &indexManager{ support: s, - ilm: ilm, + ilm: s.ilm.Manager(clientHandler), clientHandler: clientHandler, assets: assets, } @@ -177,71 +213,73 @@ func (s *indexSupport) BuildSelector(cfg *common.Config) (outputs.IndexSelector, }, nil } +func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string) { + ilmComponent := newFeature(componentILM, m.support.enabled(componentILM), false, loadILM) + + templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), + m.support.templateCfg.Overwrite, loadTemplate) + + if ilmComponent.load && !templateComponent.load { + return false, "Loading ILM policy and write alias without loading template " + + "is not recommended. Check your configuration." + } + + if templateComponent.load && !ilmComponent.load && ilmComponent.enabled { + return false, "Loading template with ILM settings whithout loading ILM " + + "policy and alias can lead to issues and is not recommended. " + + "Check your configuration." + } + + var warn string + if !ilmComponent.load { + warn += "ILM policy and write alias loading not enabled. " + } + if !templateComponent.load { + warn += "Template loading not enabled." + } + return warn == "", warn +} + +// func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { - var err error log := m.support.log - withILM := m.support.st.withILM.Load() - if !withILM { - withILM, err = m.ilm.Enabled() - if err != nil { - return err - } + withILM, err := m.setupWithILM() + if err != nil { + return err } - if loadILM == LoadModeUnset { - if withILM { - loadILM = LoadModeEnabled - log.Info("Auto ILM enable success.") - } else { - loadILM = LoadModeDisabled - } + if withILM && loadILM.Enabled() { + log.Info("Auto ILM enable success.") } - if loadILM == LoadModeForce || withILM && loadILM.Enabled() { - // mark ILM as enabled in indexState if withILM is true - m.support.st.withILM.CAS(false, true) + ilmComponent := newFeature(componentILM, withILM, false, loadILM) + templateComponent := newFeature(componentTemplate, m.support.enabled(componentTemplate), + m.support.templateCfg.Overwrite, loadTemplate) + if ilmComponent.load { // install ilm policy - policyCreated, err := m.ilm.EnsurePolicy(loadILM >= LoadModeOverwrite) + policyCreated, err := m.ilm.EnsurePolicy(ilmComponent.overwrite) if err != nil { return err } log.Info("ILM policy successfully loaded.") // The template should be updated if a new policy is created. - if policyCreated && loadTemplate.Enabled() { - loadTemplate = LoadModeOverwrite - } - - // create alias - if err := m.ilm.EnsureAlias(); err != nil { - if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists { - return err - } - log.Info("Write alias exists already") - } else { - log.Info("Write alias successfully generated.") + if policyCreated && templateComponent.enabled { + templateComponent.overwrite = true } } - // create and install template - if loadTemplate == LoadModeForce || m.support.templateCfg.Enabled && loadTemplate.Enabled() { + if templateComponent.load { tmplCfg := m.support.templateCfg + tmplCfg.Overwrite, tmplCfg.Enabled = templateComponent.overwrite, templateComponent.enabled - if withILM { - ilm := m.support.ilm - tmplCfg, err = applyILMSettings(log, tmplCfg, ilm.Policy(), ilm.Alias()) + if ilmComponent.enabled { + tmplCfg, err = applyILMSettings(log, tmplCfg, m.support.ilm.Policy(), m.support.ilm.Alias()) if err != nil { return err } } - - if loadTemplate == LoadModeForce { - tmplCfg.Enabled = true - } - if loadTemplate >= LoadModeOverwrite { - tmplCfg.Overwrite = true - } fields := m.assets.Fields(m.support.info.Beat) err = m.clientHandler.Load(tmplCfg, m.support.info, fields, m.support.migration) if err != nil { @@ -251,9 +289,37 @@ func (m *indexManager) Setup(loadTemplate, loadILM LoadMode) error { log.Info("Loaded index template.") } + if ilmComponent.load { + // ensure alias is created after the template is created + if err := m.ilm.EnsureAlias(); err != nil { + if ilm.ErrReason(err) != ilm.ErrAliasAlreadyExists { + return err + } + log.Info("Write alias exists already") + } else { + log.Info("Write alias successfully generated.") + } + } + return nil } +func (m *indexManager) setupWithILM() (bool, error) { + var err error + withILM := m.support.st.withILM.Load() + if !withILM { + withILM, err = m.ilm.Enabled() + if err != nil { + return false, err + } + if withILM { + // mark ILM as enabled in indexState + m.support.st.withILM.CAS(false, true) + } + } + return withILM, nil +} + func (s *ilmIndexSelector) Select(evt *beat.Event) (string, error) { if idx := getEventCustomIndex(evt); idx != "" { return idx, nil diff --git a/libbeat/idxmgmt/idxmgmt_test.go b/libbeat/idxmgmt/std_test.go similarity index 69% rename from libbeat/idxmgmt/idxmgmt_test.go rename to libbeat/idxmgmt/std_test.go index 01705dfa7a1..9a0b58c266c 100644 --- a/libbeat/idxmgmt/idxmgmt_test.go +++ b/libbeat/idxmgmt/std_test.go @@ -32,14 +32,32 @@ import ( "github.com/elastic/beats/libbeat/template" ) +type mockClientHandler struct { + alias, policy string + expectsPolicy bool + + tmplCfg *template.TemplateConfig + tmplForce bool + + operations []mockCreateOp +} + +type mockCreateOp uint8 + +const ( + mockCreatePolicy mockCreateOp = iota + mockCreateTemplate + mockCreateAlias +) + func TestDefaultSupport_Enabled(t *testing.T) { cases := map[string]struct { ilmCalls []onCall cfg map[string]interface{} - want bool + enabled bool }{ "templates and ilm disabled": { - want: false, + enabled: false, ilmCalls: []onCall{ onMode().Return(ilm.ModeDisabled), }, @@ -48,7 +66,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { }, }, "templates only": { - want: true, + enabled: true, ilmCalls: []onCall{ onMode().Return(ilm.ModeDisabled), }, @@ -57,7 +75,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { }, }, "ilm only": { - want: true, + enabled: true, ilmCalls: []onCall{ onMode().Return(ilm.ModeEnabled), }, @@ -66,7 +84,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { }, }, "ilm tentatively": { - want: true, + enabled: true, ilmCalls: []onCall{ onMode().Return(ilm.ModeAuto), }, @@ -81,7 +99,7 @@ func TestDefaultSupport_Enabled(t *testing.T) { factory := MakeDefaultSupport(makeMockILMSupport(test.ilmCalls...)) im, err := factory(nil, info, common.MustNewConfigFrom(test.cfg)) require.NoError(t, err) - assert.Equal(t, test.want, im.Enabled()) + assert.Equal(t, test.enabled, im.Enabled()) }) } } @@ -197,7 +215,61 @@ func TestDefaultSupport_BuildSelector(t *testing.T) { } } -func TestDefaultSupport_TemplateHandling(t *testing.T) { +func TestIndexManager_VerifySetup(t *testing.T) { + for name, setup := range map[string]struct { + tmpl, ilm bool + loadTmpl, loadILM LoadMode + ok bool + warn string + }{ + "load template with ilm without loading ilm": { + ilm: true, tmpl: true, loadILM: LoadModeDisabled, + warn: "whithout loading ILM policy and alias", + }, + "load ilm without template": { + ilm: true, loadILM: LoadModeUnset, + warn: "without loading template is not recommended", + }, + "template disabled but loading enabled": { + loadTmpl: LoadModeEnabled, + warn: "loading not enabled", + }, + "ilm disabled but loading enabled": { + loadILM: LoadModeEnabled, tmpl: true, + warn: "loading not enabled", + }, + "ilm enabled but loading disabled": { + ilm: true, loadILM: LoadModeDisabled, + warn: "loading not enabled", + }, + "template enabled but loading disabled": { + tmpl: true, loadTmpl: LoadModeDisabled, + warn: "loading not enabled", + }, + "everything enabled": { + tmpl: true, loadTmpl: LoadModeUnset, ilm: true, loadILM: LoadModeUnset, + ok: true, + }, + } { + t.Run(name, func(t *testing.T) { + cfg, err := common.NewConfigFrom(common.MapStr{ + "setup.ilm.enabled": setup.ilm, + "setup.template.enabled": setup.tmpl, + }) + require.NoError(t, err) + support, err := MakeDefaultSupport(ilm.StdSupport)(nil, beat.Info{}, cfg) + require.NoError(t, err) + clientHandler := newMockClientHandler() + manager := support.Manager(clientHandler, nil) + ok, warn := manager.VerifySetup(setup.loadTmpl, setup.loadILM) + assert.Equal(t, setup.ok, ok) + assert.Contains(t, warn, setup.warn) + clientHandler.assertInvariants(t) + }) + } +} + +func TestIndexManager_Setup(t *testing.T) { cloneCfg := func(c template.TemplateConfig) template.TemplateConfig { if c.AppendFields != nil { tmp := make(mapping.Fields, len(c.AppendFields)) @@ -235,7 +307,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { tmplCfg *template.TemplateConfig alias, policy string }{ - "template default, ilm default": { + "template default ilm default": { tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ "overwrite": "true", "name": "test-9.9.9", @@ -246,7 +318,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { alias: "test-9.9.9", policy: "test-9.9.9", }, - "template default, ilm default with alias and policy changed": { + "template default ilm default with alias and policy changed": { cfg: common.MapStr{ "setup.ilm.rollover_alias": "mocktest", "setup.ilm.policy_name": "policy-keep", @@ -261,14 +333,14 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { alias: "mocktest", policy: "policy-keep", }, - "template default, ilm disabled": { + "template default ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, loadTemplate: LoadModeEnabled, tmplCfg: &defaultCfg, }, - "template default loadMode Overwrite, ilm disabled": { + "template default loadMode Overwrite ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, @@ -277,7 +349,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { "overwrite": "true", }), }, - "template default loadMode Force, ilm disabled": { + "template default loadMode Force ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, @@ -286,27 +358,27 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { "overwrite": "true", }), }, - "template loadMode disabled, ilm disabled": { + "template loadMode disabled ilm disabled": { cfg: common.MapStr{ "setup.ilm.enabled": false, }, loadTemplate: LoadModeDisabled, }, - "template disabled, ilm default": { + "template disabled ilm default": { cfg: common.MapStr{ "setup.template.enabled": false, }, alias: "test-9.9.9", policy: "test-9.9.9", }, - "template disabled, ilm disabled, loadMode Overwrite": { + "template disabled ilm disabled, loadMode Overwrite": { cfg: common.MapStr{ "setup.template.enabled": false, "setup.ilm.enabled": false, }, loadILM: LoadModeOverwrite, }, - "template disabled, ilm disabled, loadMode Force": { + "template disabled ilm disabled loadMode Force": { cfg: common.MapStr{ "setup.template.enabled": false, "setup.ilm.enabled": false, @@ -315,13 +387,13 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { alias: "test-9.9.9", policy: "test-9.9.9", }, - "template loadmode disabled, ilm loadMode enabled": { + "template loadmode disabled ilm loadMode enabled": { loadTemplate: LoadModeDisabled, loadILM: LoadModeEnabled, alias: "test-9.9.9", policy: "test-9.9.9", }, - "template default, ilm loadMode disabled": { + "template default ilm loadMode disabled": { loadILM: LoadModeDisabled, tmplCfg: cfgWith(template.DefaultConfig(), map[string]interface{}{ "name": "test-9.9.9", @@ -330,7 +402,7 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { "settings.index.lifecycle.rollover_alias": "test-9.9.9", }), }, - "template loadmode disabled, ilm loadmode disabled": { + "template loadmode disabled ilm loadmode disabled": { loadTemplate: LoadModeDisabled, loadILM: LoadModeDisabled, }, @@ -345,69 +417,77 @@ func TestDefaultSupport_TemplateHandling(t *testing.T) { clientHandler := newMockClientHandler() manager := im.Manager(clientHandler, BeatsAssets([]byte("testbeat fields"))) err = manager.Setup(test.loadTemplate, test.loadILM) + clientHandler.assertInvariants(t) if test.err { assert.Error(t, err) } else { require.NoError(t, err) if test.tmplCfg == nil { - assert.Nil(t, clientHandler.tl.tmplCfg) + assert.Nil(t, clientHandler.tmplCfg) + } else { - assert.Equal(t, test.tmplCfg, clientHandler.tl.tmplCfg) + assert.Equal(t, test.tmplCfg, clientHandler.tmplCfg) } - assert.Equal(t, test.alias, clientHandler.il.alias) - assert.Equal(t, test.policy, clientHandler.il.policy) + assert.Equal(t, test.alias, clientHandler.alias) + assert.Equal(t, test.policy, clientHandler.policy) } }) } } -func newMockClientHandler() *mockClientHandler { - tl := mockTemplateLoader{} - il := mockILMClientHandler{} - return &mockClientHandler{&il, &tl, &tl, &il} +func (op mockCreateOp) String() string { + names := []string{"create-policy", "create-template", "create-alias"} + if int(op) > len(names) { + return "unknown" + } + return names[op] } -type mockClientHandler struct { - ilm.ClientHandler - template.Loader - - tl *mockTemplateLoader - il *mockILMClientHandler +func newMockClientHandler() *mockClientHandler { + return &mockClientHandler{} } -type mockTemplateLoader struct { - tmplCfg *template.TemplateConfig - force bool +func (h *mockClientHandler) Load(config template.TemplateConfig, _ beat.Info, fields []byte, migration bool) error { + h.recordOp(mockCreateTemplate) + h.tmplForce = config.Overwrite + h.tmplCfg = &config + return nil } -func (l *mockTemplateLoader) Load(config template.TemplateConfig, _ beat.Info, fields []byte, migration bool) error { - l.force = config.Overwrite - l.tmplCfg = &config - return nil +func (h *mockClientHandler) CheckILMEnabled(m ilm.Mode) (bool, error) { + return m == ilm.ModeEnabled || m == ilm.ModeAuto, nil } -type mockILMClientHandler struct { - alias, policy string +func (h *mockClientHandler) HasAlias(name string) (bool, error) { + return h.alias == name, nil } -func (ch *mockILMClientHandler) CheckILMEnabled(m ilm.Mode) (bool, error) { - return m == ilm.ModeEnabled || m == ilm.ModeAuto, nil +func (h *mockClientHandler) CreateAlias(alias ilm.Alias) error { + h.recordOp(mockCreateAlias) + h.alias = alias.Name + return nil } -func (ch *mockILMClientHandler) HasAlias(name string) (bool, error) { - return ch.alias == name, nil +func (h *mockClientHandler) HasILMPolicy(name string) (bool, error) { + return h.policy == name, nil } -func (ch *mockILMClientHandler) CreateAlias(alias ilm.Alias) error { - ch.alias = alias.Name +func (h *mockClientHandler) CreateILMPolicy(policy ilm.Policy) error { + h.recordOp(mockCreatePolicy) + h.policy = policy.Name return nil } -func (ch *mockILMClientHandler) HasILMPolicy(name string) (bool, error) { - return ch.policy == name, nil +func (h *mockClientHandler) recordOp(op mockCreateOp) { + h.operations = append(h.operations, op) } -func (ch *mockILMClientHandler) CreateILMPolicy(policy ilm.Policy) error { - ch.policy = policy.Name - return nil +func (h *mockClientHandler) assertInvariants(t *testing.T) { + for i, op := range h.operations { + for _, older := range h.operations[:i] { + if older > op { + t.Errorf("Operation: '%v' has been executed before '%v'", older, op) + } + } + } } diff --git a/libbeat/tests/system/idxmgmt.py b/libbeat/tests/system/idxmgmt.py index caaa7d7d974..836a6d11797 100644 --- a/libbeat/tests/system/idxmgmt.py +++ b/libbeat/tests/system/idxmgmt.py @@ -7,17 +7,20 @@ class IdxMgmt(object): def __init__(self, client, index): self._client = client - if index == "": - index == "mockbeat" - self._index = index + self._index = index if index != '' and index != '*' else 'mockbeat' def needs_init(self, s): return s == '' or s == '*' - def delete(self, index="", template="", policy=""): - self.delete_index_and_alias(index=index) - self.delete_template(template=template) - self.delete_policy(policy=policy) + def delete(self, indices=[]): + indices = list(filter(lambda x: x != '', indices)) + if not indices: + indices == [self._index] + for i in indices: + self.delete_index_and_alias(i) + self.delete_template(template=i) + for i in indices: + self.delete_policy(policy=i) def delete_index_and_alias(self, index=""): if self.needs_init(index): diff --git a/libbeat/tests/system/test_cmd_setup_index_management.py b/libbeat/tests/system/test_cmd_setup_index_management.py new file mode 100644 index 00000000000..8eab7ff3e0b --- /dev/null +++ b/libbeat/tests/system/test_cmd_setup_index_management.py @@ -0,0 +1,236 @@ +from base import BaseTest +from idxmgmt import IdxMgmt +import os +from nose.plugins.attrib import attr +import unittest +import logging +from nose.tools import raises +from elasticsearch import RequestError + +INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) + + +class TestCommandSetupIndexManagement(BaseTest): + """ + Test beat command `setup` related to ILM policy + """ + + def setUp(self): + super(TestCommandSetupIndexManagement, self).setUp() + + self.cmd = "--index-management" + # auto-derived default settings, if nothing else is set + self.index_name = self.alias_name = self.policy_name = self.beat_name + "-9.9.9" + + self.custom_alias = self.beat_name + "_foo" + self.custom_policy = self.beat_name + "_bar" + self.custom_template = self.beat_name + "_foobar" + + self.es = self.es_client() + self.idxmgmt = IdxMgmt(self.es, self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("elasticsearch").setLevel(logging.ERROR) + + def tearDown(self): + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) + + def render_config(self, **kwargs): + self.render_config_template( + elasticsearch={"hosts": self.get_elasticsearch_url()}, + es_template_name=self.index_name, + **kwargs + ) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_default(self): + """ + Test setup --index-management with default config + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_policy_created(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + @raises(RequestError) + def test_setup_default(self): + """ + Test setup --index-management with default config + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.index_name, [self.index_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + self.idxmgmt.assert_policy_created(self.policy_name) + # try deleting policy needs to raise an error as it is in use + self.idxmgmt.delete_policy(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_disabled(self): + """ + Test setup --index-management when ilm disabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.template.enabled=false"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_not_loaded(self.index_name) + self.idxmgmt.assert_alias_created(self.index_name) + self.idxmgmt.assert_policy_created(self.index_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_ilm_disabled(self): + """ + Test setup --index-management when ilm disabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_policy_not_created(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_policy_name(self): + """ + Test setup --index-management when policy_name is configured + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.policy_name=" + self.custom_policy]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.custom_policy, self.alias_name) + self.idxmgmt.assert_policy_created(self.custom_policy) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_rollover_alias(self): + """ + Test setup --index-management when ilm.rollover_alias is configured + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.policy_name, self.custom_alias) + self.idxmgmt.assert_index_template_index_pattern(self.custom_alias, [self.custom_alias + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.custom_alias) + self.idxmgmt.assert_alias_created(self.custom_alias) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_name_and_pattern(self): + """ + Test setup --index-management ignores template.name and template.pattern when ilm is enabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.template.name=" + self.custom_template, + "-E", "setup.template.pattern=" + self.custom_template + "*"]) + + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"]) + self.idxmgmt.assert_docs_written_to_alias(self.alias_name) + self.idxmgmt.assert_alias_created(self.alias_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_name_and_pattern_on_ilm_disabled(self): + """ + Test setup --index-management respects template.name and template.pattern when ilm is disabled + """ + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false", + "-E", "setup.template.name=" + self.custom_template, + "-E", "setup.template.pattern=" + self.custom_template + "*"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.custom_template) + self.idxmgmt.assert_index_template_index_pattern(self.custom_template, [self.custom_template + "*"]) + self.idxmgmt.assert_alias_not_created(self.alias_name) + self.idxmgmt.assert_policy_not_created(self.policy_name) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_template_with_opts(self): + """ + Test setup --index-management with config options + """ + self.render_config() + + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false", + "-E", "setup.template.settings.index.number_of_shards=2"]) + + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.index_name) + + # check that settings are overwritten + resp = self.es.transport.perform_request('GET', '/_template/' + self.index_name) + assert self.index_name in resp + index = resp[self.index_name]["settings"]["index"] + assert index["number_of_shards"] == "2", index["number_of_shards"] + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_overwrite_template_on_ilm_policy_created(self): + """ + Test setup --index-management overwrites template when new ilm policy is created + """ + + # ensure template with ilm rollover_alias name is created, but ilm policy not yet + self.render_config() + exit_code = self.run_beat(logging_args=["-v", "-d", "*"], + extra_args=["setup", self.cmd, + "-E", "setup.ilm.enabled=false", + "-E", "setup.template.name=" + self.custom_alias, + "-E", "setup.template.pattern=" + self.custom_alias + "*"]) + assert exit_code == 0 + self.idxmgmt.assert_index_template_loaded(self.custom_alias) + self.idxmgmt.assert_policy_not_created(self.index_name) + + # ensure ilm policy is created, triggering overwriting existing template + exit_code = self.run_beat(extra_args=["setup", self.cmd, + "-E", "setup.template.overwrite=false", + "-E", "setup.template.settings.index.number_of_shards=2", + "-E", "setup.ilm.rollover_alias=" + self.custom_alias]) + assert exit_code == 0 + self.idxmgmt.assert_ilm_template_loaded(self.custom_alias, self.index_name, self.custom_alias) + self.idxmgmt.assert_policy_created(self.index_name) + # check that template was overwritten + resp = self.es.transport.perform_request('GET', '/_template/' + self.custom_alias) + assert self.custom_alias in resp + index = resp[self.custom_alias]["settings"]["index"] + assert index["number_of_shards"] == "2", index["number_of_shards"] diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py index 2dcc11edd6f..9195954600a 100644 --- a/libbeat/tests/system/test_ilm.py +++ b/libbeat/tests/system/test_ilm.py @@ -6,6 +6,7 @@ import shutil import datetime import logging +import json INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -20,14 +21,10 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( @@ -168,17 +165,13 @@ def setUp(self): self.custom_policy = self.beat_name + "_bar" self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.custom_policy) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name, self.custom_policy]) def render_config(self, **kwargs): self.render_config_template( @@ -217,6 +210,7 @@ def test_setup_ilm_default(self): assert exit_code == 0 self.idxmgmt.assert_ilm_template_loaded(self.alias_name, self.policy_name, self.alias_name) + self.idxmgmt.assert_index_template_index_pattern(self.alias_name, [self.alias_name + "-*"]) self.idxmgmt.assert_docs_written_to_alias(self.alias_name) self.idxmgmt.assert_alias_created(self.alias_name) self.idxmgmt.assert_policy_created(self.policy_name) @@ -321,7 +315,6 @@ def test_load_disabled(self): def test_changed_policy_name(self): """ Test ilm-policy export when policy name is changed - """ policy_name = "foo" @@ -331,3 +324,42 @@ def test_changed_policy_name(self): assert exit_code == 0 self.assert_log_contains_policy() self.assert_log_contains_write_alias() + + def test_export_to_file_absolute_path(self): + """ + Test export ilm policy to file with absolute file path + """ + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + exit_code = self.run_beat( + extra_args=["export", self.cmd, "--dir=" + base_path], + config=self.config) + + assert exit_code == 0 + + file = os.path.join(base_path, "policy", self.policy_name + '.json') + with open(file) as f: + policy = json.load(f) + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_size"] == "50gb", policy + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_age"] == "30d", policy + + os.remove(file) + + def test_export_to_file_relative_path(self): + """ + Test export ilm policy to file with relative file path + """ + path = os.path.join(os.path.dirname(__file__), "export") + exit_code = self.run_beat( + extra_args=["export", self.cmd, "--dir=" + path], + config=self.config) + + assert exit_code == 0 + + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + file = os.path.join(base_path, "policy", self.policy_name + '.json') + with open(file) as f: + policy = json.load(f) + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_size"] == "50gb", policy + assert policy["policy"]["phases"]["hot"]["actions"]["rollover"]["max_age"] == "30d", policy + + os.remove(file) diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index 962f5aed419..084993b8850 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -5,6 +5,7 @@ import unittest import shutil import logging +import json INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) @@ -88,17 +89,18 @@ def test_json_template(self): Test loading of json based template """ + template_name = "bla" + es = self.es_client() self.copy_files(["template.json"]) - path = os.path.join(self.working_dir, "template.json") - print path + self.render_config_template( elasticsearch={"hosts": self.get_host()}, template_overwrite="true", template_json_enabled="true", template_json_path=path, - template_json_name="bla", + template_json_name=template_name, ) proc = self.start_beat() @@ -107,8 +109,7 @@ def test_json_template(self): self.wait_until(lambda: self.log_contains("template with name 'bla' loaded")) proc.check_kill_and_wait() - es = self.es_client() - result = es.transport.perform_request('GET', '/_template/bla') + result = es.transport.perform_request('GET', '/_template/' + template_name) assert len(result) == 1 def get_host(self): @@ -127,10 +128,10 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.index_name]) def tearDown(self): - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -185,14 +186,12 @@ def setUp(self): self.es = self.es_client() self.idxmgmt = IdxMgmt(self.es, self.index_name) - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("elasticsearch").setLevel(logging.ERROR) def tearDown(self): - self.idxmgmt.delete(index=self.custom_alias) - self.idxmgmt.delete(index=self.index_name) + self.idxmgmt.delete(indices=[self.custom_alias, self.index_name]) def render_config(self, **kwargs): self.render_config_template( @@ -335,9 +334,8 @@ def setUp(self): shutil.copy(os.path.join(self.beat_path, "fields.yml"), self.output) self.template_name = self.beat_name + "-9.9.9" - def assert_log_contains_template(self, template, index_pattern): + def assert_log_contains_template(self, index_pattern): assert self.log_contains('Loaded index template') - assert self.log_contains(template) assert self.log_contains(index_pattern) def test_default(self): @@ -351,7 +349,7 @@ def test_default(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name, self.template_name + "-*") + self.assert_log_contains_template(self.template_name + "-*") def test_changed_index_pattern(self): """ @@ -367,7 +365,7 @@ def test_changed_index_pattern(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name, alias_name + "-*") + self.assert_log_contains_template(alias_name + "-*") def test_load_disabled(self): """ @@ -380,4 +378,49 @@ def test_load_disabled(self): config=self.config) assert exit_code == 0 - self.assert_log_contains_template(self.template_name, self.template_name + "-*") + self.assert_log_contains_template(self.template_name + "-*") + + def test_export_to_file_absolute_path(self): + """ + Test export template to file with absolute file path + """ + self.render_config_template(self.beat_name, self.output, + fields=self.output) + + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + exit_code = self.run_beat( + extra_args=["export", "template", "--dir=" + base_path], + config=self.config) + + assert exit_code == 0 + + file = os.path.join(base_path, "template", self.template_name + '.json') + with open(file) as f: + template = json.load(f) + assert 'index_patterns' in template + assert template['index_patterns'] == [self.template_name + '-*'], template + + os.remove(file) + + def test_export_to_file_relative_path(self): + """ + Test export template to file with relative file path + """ + self.render_config_template(self.beat_name, self.output, + fields=self.output) + + path = os.path.join(os.path.dirname(__file__), "export") + exit_code = self.run_beat( + extra_args=["export", "template", "--dir=" + path], + config=self.config) + + assert exit_code == 0 + + base_path = os.path.abspath(os.path.join(self.beat_path, os.path.dirname(__file__), "export")) + file = os.path.join(base_path, "template", self.template_name + '.json') + with open(file) as f: + template = json.load(f) + assert 'index_patterns' in template + assert template['index_patterns'] == [self.template_name + '-*'], template + + os.remove(file)