Move default config logic to the builder

Autodiscover hints default config (unreleased) was done at the hints
level, while it only makes sense for certain configs builder (logs).

This change moves this logic to the `logs` builder and improves how
users can define default settings for logs, ie:

Enable hints but disable log retrieval by default:

```
filebeat.autodiscover:
  providers:
    - type: docker # or kubernetes
      hints.enabled: true
      hints.default_config.enabled: false
```

Only containers with the `co.elastic.logs/enabled: true` annotation
will be retrieved.

```
filebeat.autodiscover:
  providers:
    - type: kubernetes # or docker
      hints.enabled: true
      hints.default_config:
        type: container
        paths:
          - /var/log/containers/*${data.container.id}.log
```

Logs are read by default for all containers, using the given input
settings. Containers with `co.elastic.logs/enabled: false` annotation
will be ignored.

filebeat/autodiscover/builder/hints/config.go

@@ -20,12 +20,12 @@ package hints
 import "github.com/elastic/beats/libbeat/common"
 
 type config struct {
-	Key    string         `config:"key"`
-	Config *common.Config `config:"config"`
+	Key           string         `config:"key"`
+	DefaultConfig *common.Config `config:"default_config"`
 }
 
 func defaultConfig() config {
-	rawCfg := map[string]interface{}{
+	defaultCfgRaw := map[string]interface{}{
 		"type": "container",
 		"containers": map[string]interface{}{
 			"paths": []string{
@@ -35,10 +35,10 @@ func defaultConfig() config {
 			},
 		},
 	}
-	cfg, _ := common.NewConfigFrom(rawCfg)
+	defaultCfg, _ := common.NewConfigFrom(defaultCfgRaw)
 	return config{
-		Key:    "logs",
-		Config: cfg,
+		Key:           "logs",
+		DefaultConfig: defaultCfg,
 	}
 }
 
@@ -52,8 +52,8 @@ func (c *config) Unpack(from *common.Config) error {
 		return err
 	}
 
-	if config, err := from.Child("config", -1); err == nil {
-		c.Config = config
+	if config, err := from.Child("default_config", -1); err == nil {
+		c.DefaultConfig = config
 	}
 
 	c.Key = tmpConfig.Key

filebeat/autodiscover/builder/hints/logs.go

@@ -27,7 +27,6 @@ import (
 	"github.com/elastic/beats/libbeat/autodiscover/template"
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/common/bus"
-	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/logp"
 )
 
@@ -46,14 +45,12 @@ const (
 var validModuleNames = regexp.MustCompile("[^a-zA-Z0-9\\_\\-]+")
 
 type logHints struct {
-	Key      string
-	Config   *common.Config
-	Registry *fileset.ModuleRegistry
+	config   *config
+	registry *fileset.ModuleRegistry
 }
 
 // NewLogHints builds a log hints builder
 func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) {
-	cfgwarn.Beta("The hints builder is beta")
 	config := defaultConfig()
 	err := cfg.Unpack(&config)
 
@@ -66,30 +63,38 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) {
 		return nil, err
 	}
 
-	return &logHints{config.Key, config.Config, moduleRegistry}, nil
+	return &logHints{&config, moduleRegistry}, nil
 }
 
 // Create config based on input hints in the bus event
 func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
-	// Clone original config
-	config, _ := common.NewConfigFrom(l.Config)
-	host, _ := event["host"].(string)
-	if host == "" {
-		return []*common.Config{}
-	}
-
 	var hints common.MapStr
 	hIface, ok := event["hints"]
 	if ok {
 		hints, _ = hIface.(common.MapStr)
 	}
 
-	if builder.IsNoOp(hints, l.Key) {
-		logp.Debug("hints.builder", "disabled config in event: %+v", event)
+	inputConfig := l.getInputs(hints)
+
+	// If default config is disabled return nothing unless it's explicty enabled
+	if !l.config.DefaultConfig.Enabled() && !builder.IsEnabled(hints, l.config.Key) {
+		logp.Debug("hints.builder", "default config is disabled: %+v", event)
+		return []*common.Config{}
+	}
+
+	// If explicty disabled, return nothing
+	if builder.IsDisabled(hints, l.config.Key) {
+		logp.Debug("hints.builder", "logs disabled by hint: %+v", event)
+		return []*common.Config{}
+	}
+
+	// Clone original config
+	config, _ := common.NewConfigFrom(l.config.DefaultConfig)
+	host, _ := event["host"].(string)
+	if host == "" {
 		return []*common.Config{}
 	}
 
-	inputConfig := l.getInputs(hints)
 	if inputConfig != nil {
 		configs := []*common.Config{}
 		for _, cfg := range inputConfig {
@@ -149,29 +154,29 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 }
 
 func (l *logHints) getMultiline(hints common.MapStr) common.MapStr {
-	return builder.GetHintMapStr(hints, l.Key, multiline)
+	return builder.GetHintMapStr(hints, l.config.Key, multiline)
 }
 
 func (l *logHints) getIncludeLines(hints common.MapStr) []string {
-	return builder.GetHintAsList(hints, l.Key, includeLines)
+	return builder.GetHintAsList(hints, l.config.Key, includeLines)
 }
 
 func (l *logHints) getExcludeLines(hints common.MapStr) []string {
-	return builder.GetHintAsList(hints, l.Key, excludeLines)
+	return builder.GetHintAsList(hints, l.config.Key, excludeLines)
 }
 
 func (l *logHints) getModule(hints common.MapStr) string {
-	module := builder.GetHintString(hints, l.Key, "module")
+	module := builder.GetHintString(hints, l.config.Key, "module")
 	// for security, strip module name
 	return validModuleNames.ReplaceAllString(module, "")
 }
 
 func (l *logHints) getInputs(hints common.MapStr) []common.MapStr {
-	return builder.GetHintAsConfigs(hints, l.Key)
+	return builder.GetHintAsConfigs(hints, l.config.Key)
 }
 
 func (l *logHints) getProcessors(hints common.MapStr) []common.MapStr {
-	return builder.GetProcessors(hints, l.Key)
+	return builder.GetProcessors(hints, l.config.Key)
 }
 
 type filesetConfig struct {
@@ -184,7 +189,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 	var configured bool
 	filesets := make(map[string]*filesetConfig)
 
-	moduleFilesets, err := l.Registry.ModuleFilesets(module)
+	moduleFilesets, err := l.registry.ModuleFilesets(module)
 	if err != nil {
 		logp.Err("Error retrieving module filesets: %+v", err)
 		return nil
@@ -195,7 +200,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 	}
 
 	// If a single fileset is given, pass all streams to it
-	fileset := builder.GetHintString(hints, l.Key, "fileset")
+	fileset := builder.GetHintString(hints, l.config.Key, "fileset")
 	if fileset != "" {
 		if conf, ok := filesets[fileset]; ok {
 			conf.Enabled = true
@@ -205,7 +210,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f
 
 	// If fileset is defined per stream, return all of them
 	for _, stream := range []string{"all", "stdout", "stderr"} {
-		fileset := builder.GetHintString(hints, l.Key, "fileset."+stream)
+		fileset := builder.GetHintString(hints, l.config.Key, "fileset."+stream)
 		if fileset != "" {
 			if conf, ok := filesets[fileset]; ok {
 				conf.Enabled = true

filebeat/autodiscover/builder/hints/logs_test.go

@@ -377,7 +377,7 @@ func TestGenerateHints(t *testing.T) {
 
 	for _, test := range tests {
 		cfg, _ := common.NewConfigFrom(map[string]interface{}{
-			"config": map[string]interface{}{
+			"default_config": map[string]interface{}{
 				"type": "docker",
 				"containers": map[string]interface{}{
 					"ids": []string{
@@ -609,7 +609,7 @@ func TestGenerateHintsWithPaths(t *testing.T) {
 
 	for _, test := range tests {
 		cfg, _ := common.NewConfigFrom(map[string]interface{}{
-			"config": map[string]interface{}{
+			"default_config": map[string]interface{}{
 				"type": "docker",
 				"containers": map[string]interface{}{
 					"paths": []string{

filebeat/input/docker/input.go

@@ -68,7 +68,7 @@ func NewInput(
 	}
 
 	if len(ids) == 0 {
-		return nil, errors.New("Docker input requires at least one entry under 'containers.ids''")
+		return nil, errors.New("Docker input requires at least one entry under 'containers.ids' or 'containers.paths'")
 	}
 
 	for idx, containerID := range ids {

libbeat/autodiscover/builder.go

@@ -99,21 +99,18 @@ func (b Builders) GetConfig(event bus.Event) []*common.Config {
 	return configs
 }
 
-// NewBuilders instances the given list of builders. If hintsEnabled is true it will
-// just enable the hints builder
-func NewBuilders(bConfigs []*common.Config, hintsEnabled bool) (Builders, error) {
+// NewBuilders instances the given list of builders. hintsCfg holds `hints` settings
+// for simplified mode (single 'hints' builder)
+func NewBuilders(bConfigs []*common.Config, hintsCfg *common.Config) (Builders, error) {
 	var builders Builders
-	if hintsEnabled {
+	if hintsCfg.Enabled() {
 		if len(bConfigs) > 0 {
 			return nil, errors.New("hints.enabled is incompatible with manually defining builders")
 		}
 
-		hints, err := common.NewConfigFrom(map[string]string{"type": "hints"})
-		if err != nil {
-			return nil, err
-		}
-
-		bConfigs = append(bConfigs, hints)
+		// pass rest of hints settings to the builder
+		hintsCfg.SetString("type", -1, "hints")
+		bConfigs = append(bConfigs, hintsCfg)
 	}
 
 	for _, bcfg := range bConfigs {

libbeat/autodiscover/builder/helper.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 
 	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/libbeat/logp"
 )
 
@@ -144,18 +145,38 @@ func GetHintAsConfigs(hints common.MapStr, key string) []common.MapStr {
 	return nil
 }
 
-// IsNoOp is a big red button to prevent spinning up Runners in case of issues.
-func IsNoOp(hints common.MapStr, key string) bool {
+// IsEnabled will return true when 'enabled' is **explicity** set to true
+func IsEnabled(hints common.MapStr, key string) bool {
+	if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil {
+		enabled, _ := strconv.ParseBool(value.(string))
+		return enabled
+	}
+
+	return false
+}
+
+// IsDisabled will return true when 'enabled' key is **explicity** set to false
+func IsDisabled(hints common.MapStr, key string) bool {
+	if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil {
+		enabled, err := strconv.ParseBool(value.(string))
+		if err == nil {
+			logp.Debug("autodiscover.builder", "error parsing 'enabled' hint from: %+v", hints)
+			return !enabled
+		}
+	}
+
+	// keep reading disable (deprecated) for backwards compatibility
 	if value, err := hints.GetValue(fmt.Sprintf("%s.disable", key)); err == nil {
-		noop, _ := strconv.ParseBool(value.(string))
-		return noop
+		cfgwarn.Deprecate("8.0.0", "disable hint is deprecated. Use `enabled: false` instead.")
+		disabled, _ := strconv.ParseBool(value.(string))
+		return disabled
 	}
 
 	return false
 }
 
 // GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats.
-func GenerateHints(annotations common.MapStr, container, prefix string, defaultDisable bool) common.MapStr {
+func GenerateHints(annotations common.MapStr, container, prefix string) common.MapStr {
 	hints := common.MapStr{}
 	if rawEntries, err := annotations.GetValue(prefix); err == nil {
 		if entries, ok := rawEntries.(common.MapStr); ok {
@@ -195,10 +216,5 @@ func GenerateHints(annotations common.MapStr, container, prefix string, defaultD
 		}
 	}
 
-	// Update hints: if .disabled annotation does not exist, set according to disabledByDefault flag
-	if _, err := hints.GetValue("logs.disable"); err != nil && defaultDisable {
-		hints.Put("logs.disable", "true")
-	}
-
 	return hints
 }