Add JSON options to autodiscover hints

CHANGELOG.next.asciidoc

@@ -366,6 +366,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add module for MISP (Malware Information Sharing Platform). {pull}13805[13805]
 - Add `source.bytes` and `source.packets` for uni-directional netflow events. {pull}14111[14111]
 - Add Kibana Dashboard for MISP module. {pull}14147[14147]
+- Add JSON options to autodiscover hints {pull}14208[14208]
 
 *Heartbeat*
 - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498]

filebeat/autodiscover/builder/hints/logs.go

@@ -40,6 +40,7 @@ const (
 	includeLines = "include_lines"
 	excludeLines = "exclude_lines"
 	processors   = "processors"
+	json         = "json"
 )
 
 // validModuleNames to sanitize user input
@@ -126,6 +127,9 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 		tempCfg.Put(processors, procs)
 	}
 
+	if jsonOpts := l.getJSONOptions(hints); len(jsonOpts) != 0 {
+		tempCfg.Put(json, jsonOpts)
+	}
 	// Merge config template with the configs from the annotations
 	if err := config.Merge(tempCfg); err != nil {
 		logp.Debug("hints.builder", "config merge failed with error: %v", err)
@@ -187,6 +191,10 @@ func (l *logHints) getProcessors(hints common.MapStr) []common.MapStr {
 	return builder.GetProcessors(hints, l.config.Key)
 }
 
+func (l *logHints) getJSONOptions(hints common.MapStr) common.MapStr {
+	return builder.GetHintMapStr(hints, l.config.Key, json)
+}
+
 type filesetConfig struct {
 	Enabled bool
 	Stream  string

filebeat/docs/autodiscover-hints.asciidoc

@@ -18,6 +18,11 @@ set to `true`.
 
 Multiline settings. See <<multiline-examples>> for a full list of all supported options.
 
+[float]
+===== `co.elastic.logs/json.*`
+
+JSON settings. See <<filebeat-input-log-config-json>> for a full list of all supported options.
+
 [float]
 ===== `co.elastic.logs/include_lines`
 
@@ -202,3 +207,13 @@ You can label Docker containers with useful info to spin up {beatname_uc} inputs
 
 The above labels configure {beatname_uc} to use the Nginx module to harvest logs for this container.
 Access logs will be retrieved from stdout stream, and error logs from stderr.
+
+
+You can label Docker containers with useful info to decode logs structured as JSON messages, for example:
+
+[source,yaml]
+-----
+  co.elastic.logs/json.keys_under_root: true
+  co.elastic.logs/json.add_error_key: true
+  co.elastic.logs/json.message_key: log
+-----

libbeat/autodiscover/builder/helper_test.go

@@ -38,12 +38,14 @@ func TestGenerateHints(t *testing.T) {
 
 		// Scenarios being tested:
 		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// logs/json.keys_under_root must be a nested common.MapStr under hints.logs
 		// metrics/module must be found in hints.metrics
 		// not.to.include must not be part of hints
 		// period is annotated at both container and pod level. Container level value must be in hints
 		{
 			annotations: map[string]string{
 				"co.elastic.logs/multiline.pattern": "^test",
+				"co.elastic.logs/json.keys_under_root": "true",
 				"co.elastic.metrics/module":         "prometheus",
 				"co.elastic.metrics/period":         "10s",
 				"co.elastic.metrics.foobar/period":  "15s",
@@ -55,6 +57,9 @@ func TestGenerateHints(t *testing.T) {
 					"multiline": common.MapStr{
 						"pattern": "^test",
 					},
+					"json": common.MapStr{
+						"keys_under_root": "true",
+					},
 				},
 				"metrics": common.MapStr{
 					"module": "prometheus",

libbeat/autodiscover/providers/kubernetes/kubernetes_test.go

@@ -89,6 +89,7 @@ func TestGenerateHints(t *testing.T) {
 		},
 		// Scenarios being tested:
 		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// logs/json.keys_under_root must be a nested common.MapStr under hints.logs
 		// metrics/module must be found in hints.metrics
 		// not.to.include must not be part of hints
 		// period is annotated at both container and pod level. Container level value must be in hints
@@ -97,6 +98,7 @@ func TestGenerateHints(t *testing.T) {
 				"kubernetes": common.MapStr{
 					"annotations": getNestedAnnotations(common.MapStr{
 						"co.elastic.logs/multiline.pattern": "^test",
+						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":         "prometheus",
 						"co.elastic.metrics/period":         "10s",
 						"co.elastic.metrics.foobar/period":  "15s",
@@ -113,6 +115,7 @@ func TestGenerateHints(t *testing.T) {
 				"kubernetes": common.MapStr{
 					"annotations": getNestedAnnotations(common.MapStr{
 						"co.elastic.logs/multiline.pattern": "^test",
+						"co.elastic.logs/json.keys_under_root": "true",
 						"co.elastic.metrics/module":         "prometheus",
 						"not.to.include":                    "true",
 						"co.elastic.metrics/period":         "10s",
@@ -129,6 +132,9 @@ func TestGenerateHints(t *testing.T) {
 						"multiline": common.MapStr{
 							"pattern": "^test",
 						},
+						"json": common.MapStr{
+							"keys_under_root": "true",
+						},
 					},
 					"metrics": common.MapStr{
 						"module": "prometheus",