Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[cmd setup] Add and deprecate setup cmds for index handling #12132

Merged
merged 10 commits into from
May 11, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion CHANGELOG-developer.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
- Reduce idxmgmt.Supporter interface and rework export commands to reuse logic. {pull}11777[11777], {pull}12065[12065], {pull}12067[12067]
- Update urllib3 version to 1.24.2 {pull}11930[11930]
- Add libbeat/common/cleanup package. {pull}12134[12134]
- Only Load minimal template if no fields are provided. {pull}12103[12103]
- Only Load minimal template if no fields are provided. {pull}12103[12103]
- Deprecate setup cmds for `template` and `ilm-policy`. Add new setup cmd for `index-management`. {pull}12132[12132]
51 changes: 24 additions & 27 deletions libbeat/cmd/instance/beat.go
Original file line number Diff line number Diff line change
Expand Up @@ -447,11 +447,14 @@ func (b *Beat) TestConfig(settings Settings, bt beat.Creator) error {

//SetupSettings holds settings necessary for beat setup
type SetupSettings struct {
Template bool
Dashboard bool
MachineLearning bool
Pipeline bool
ILMPolicy bool
IndexManagement bool
//Deprecated: use IndexManagementKey instead
Template bool
//Deprecated: use IndexManagementKey instead
ILMPolicy bool
}

// Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
Expand All @@ -471,37 +474,31 @@ func (b *Beat) Setup(settings Settings, bt beat.Creator, setup SetupSettings) er
return err
}

if setup.Template || setup.ILMPolicy {
if setup.IndexManagement || setup.Template || setup.ILMPolicy {
outCfg := b.Config.Output

if outCfg.Name() != "elasticsearch" {
return fmt.Errorf("Index management requested but the Elasticsearch output is not configured/enabled")
}
esClient, err := elasticsearch.NewConnectedClient(outCfg.Config())
if err != nil {
return err
}

esConfig := outCfg.Config()
if b.IdxSupporter.Enabled() {
esClient, err := elasticsearch.NewConnectedClient(esConfig)
if err != nil {
return err
}

// prepare index by loading templates, lifecycle policies and write aliases

m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
var tmplLoadMode, ilmLoadMode = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
if setup.Template {
tmplLoadMode = idxmgmt.LoadModeOverwrite
}
if setup.ILMPolicy {
ilmLoadMode = idxmgmt.LoadModeOverwrite
}

err = m.Setup(tmplLoadMode, ilmLoadMode)
if err != nil {
return err
}
var loadTemplate, loadILM = idxmgmt.LoadModeUnset, idxmgmt.LoadModeUnset
if setup.IndexManagement || setup.Template {
loadTemplate = idxmgmt.LoadModeOverwrite
}
if setup.IndexManagement || setup.ILMPolicy {
loadILM = idxmgmt.LoadModeOverwrite
}
m := b.IdxSupporter.Manager(idxmgmt.NewESClientHandler(esClient), idxmgmt.BeatsAssets(b.Fields))
if ok, warn := m.VerifySetup(loadTemplate, loadILM); !ok {
fmt.Println(warn)
}
if err = m.Setup(loadTemplate, loadILM); err != nil {
return err
}
fmt.Println("Index setup complete.")
fmt.Println("Index setup finished.")
}

if setup.Dashboard {
Expand Down
29 changes: 22 additions & 7 deletions libbeat/cmd/setup.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,23 @@ import (
)

const (
//TemplateKey used for defining template in setup cmd
TemplateKey = "template"
//DashboardKey used for registering dashboards in setup cmd
DashboardKey = "dashboards"
//MachineLearningKey used for registering ml jobs in setup cmd
MachineLearningKey = "machine-learning"
//PipelineKey used for registering pipelines in setup cmd
PipelineKey = "pipelines"
//ILMPolicyKey used for registering ilm in setup cmd
//IndexManagementKey used for loading all components related to ES index management in setup cmd
IndexManagementKey = "index-management"

//TemplateKey used for loading template in setup cmd
//
//Deprecated: use IndexManagementKey instead
TemplateKey = "template"

//ILMPolicyKey used for loading ilm in setup cmd
//
//Deprecated: use IndexManagementKey instead
ILMPolicyKey = "ilm-policy"
)

Expand All @@ -60,10 +68,11 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
}

var registeredFlags = map[string]bool{
TemplateKey: false,
DashboardKey: false,
MachineLearningKey: false,
PipelineKey: false,
IndexManagementKey: false,
TemplateKey: false,
ILMPolicyKey: false,
}
var setupAll = true
Expand All @@ -89,16 +98,18 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
for k, v := range registeredFlags {
if setupAll || v {
switch k {
case TemplateKey:
s.Template = true
case DashboardKey:
s.Dashboard = true
case MachineLearningKey:
s.MachineLearning = true
case PipelineKey:
s.Pipeline = true
case IndexManagementKey:
s.IndexManagement = true
case ILMPolicyKey:
s.ILMPolicy = true
case TemplateKey:
s.Template = true
}
}
}
Expand All @@ -109,11 +120,15 @@ func genSetupCmd(settings instance.Settings, beatCreator beat.Creator) *cobra.Co
},
}

setup.Flags().Bool(TemplateKey, false, "Setup index template")
setup.Flags().Bool(DashboardKey, false, "Setup dashboards")
setup.Flags().Bool(MachineLearningKey, false, "Setup machine learning job configurations")
setup.Flags().Bool(PipelineKey, false, "Setup Ingest pipelines")
setup.Flags().Bool(IndexManagementKey, false,
"Setup all components related to Elasticsearch index management, including template, ilm policy and rollover alias")
setup.Flags().Bool(TemplateKey, false, "Setup index template")
setup.Flags().MarkDeprecated(TemplateKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))
setup.Flags().Bool(ILMPolicyKey, false, "Setup ILM policy")
setup.Flags().MarkDeprecated(ILMPolicyKey, fmt.Sprintf("please use --%s instead", IndexManagementKey))

return &setup
}
41 changes: 41 additions & 0 deletions libbeat/idxmgmt/componenttype_string.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions libbeat/idxmgmt/idxmgmt.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ type SupportFactory func(*logp.Logger, beat.Info, *common.Config) (Supporter, er
// A manager instantiated via Supporter is responsible for instantiating/configuring
// the index throughout the Elastic Stack.
type Supporter interface {
// Enalbed checks if index management is configured to configure templates,
// ILM, or aliases.
// Enabled checks if index management is configured to setup templates or ILM
Enabled() bool

// BuildSelector create an index selector.
Expand All @@ -62,6 +61,9 @@ type Asseter interface {
// Manager is used to initialize indices, ILM policies, and aliases within the
// Elastic Stack.
type Manager interface {
VerifySetup(template, ilm LoadMode) (bool, string)
// When supporting index lifecycle management, ensure templates and policies
// are created before write aliases, to ensure templates are applied to the indices.
Setup(template, ilm LoadMode) error
}

Expand Down
Loading