Add JSON options to autodiscover hints (elastic#14208)

jorgemarey · Dec 13, 2019 · f61bf0d · f61bf0d
1 parent 2e73533
commit f61bf0d
Show file tree

Hide file tree

Showing 5 changed files with 51 additions and 16 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -307,6 +307,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add module for MISP (Malware Information Sharing Platform). {pull}13805[13805]
 - Add `source.bytes` and `source.packets` for uni-directional netflow events. {pull}14111[14111]
 - Add Kibana Dashboard for MISP module. {pull}14147[14147]
+- Add JSON options to autodiscover hints {pull}14208[14208]
 - Add more filesets to Zeek module. {pull}14150[14150]
 
 *Heartbeat*

diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go
@@ -40,6 +40,7 @@ const (
 	includeLines = "include_lines"
 	excludeLines = "exclude_lines"
 	processors   = "processors"
+	json         = "json"
 )
 
 // validModuleNames to sanitize user input
@@ -126,6 +127,9 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 		tempCfg.Put(processors, procs)
 	}
 
+	if jsonOpts := l.getJSONOptions(hints); len(jsonOpts) != 0 {
+		tempCfg.Put(json, jsonOpts)
+	}
 	// Merge config template with the configs from the annotations
 	if err := config.Merge(tempCfg); err != nil {
 		logp.Debug("hints.builder", "config merge failed with error: %v", err)
@@ -187,6 +191,10 @@ func (l *logHints) getProcessors(hints common.MapStr) []common.MapStr {
 	return builder.GetProcessors(hints, l.config.Key)
 }
 
+func (l *logHints) getJSONOptions(hints common.MapStr) common.MapStr {
+	return builder.GetHintMapStr(hints, l.config.Key, json)
+}
+
 type filesetConfig struct {
 	Enabled bool
 	Stream  string

diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc
@@ -18,6 +18,11 @@ set to `true`.
 
 Multiline settings. See <<multiline-examples>> for a full list of all supported options.
 
+[float]
+===== `co.elastic.logs/json.*`
+
+JSON settings. See <<filebeat-input-log-config-json>> for a full list of all supported options.
+
 [float]
 ===== `co.elastic.logs/include_lines`
 
@@ -202,3 +207,13 @@ You can label Docker containers with useful info to spin up {beatname_uc} inputs
 
 The above labels configure {beatname_uc} to use the Nginx module to harvest logs for this container.
 Access logs will be retrieved from stdout stream, and error logs from stderr.
+
+
+You can label Docker containers with useful info to decode logs structured as JSON messages, for example:
+
+[source,yaml]
+-----
+  co.elastic.logs/json.keys_under_root: true
+  co.elastic.logs/json.add_error_key: true
+  co.elastic.logs/json.message_key: log
+-----
diff --git a/libbeat/autodiscover/builder/helper_test.go b/libbeat/autodiscover/builder/helper_test.go
@@ -38,23 +38,28 @@ func TestGenerateHints(t *testing.T) {
 
 		// Scenarios being tested:
 		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// logs/json.keys_under_root must be a nested common.MapStr under hints.logs
 		// metrics/module must be found in hints.metrics
 		// not.to.include must not be part of hints
 		// period is annotated at both container and pod level. Container level value must be in hints
 		{
 			annotations: map[string]string{
-				"co.elastic.logs/multiline.pattern": "^test",
-				"co.elastic.metrics/module":         "prometheus",
-				"co.elastic.metrics/period":         "10s",
-				"co.elastic.metrics.foobar/period":  "15s",
-				"co.elastic.metrics.foobar1/period": "15s",
-				"not.to.include":                    "true",
+				"co.elastic.logs/multiline.pattern":    "^test",
+				"co.elastic.logs/json.keys_under_root": "true",
+				"co.elastic.metrics/module":            "prometheus",
+				"co.elastic.metrics/period":            "10s",
+				"co.elastic.metrics.foobar/period":     "15s",
+				"co.elastic.metrics.foobar1/period":    "15s",
+				"not.to.include":                       "true",
 			},
 			result: common.MapStr{
 				"logs": common.MapStr{
 					"multiline": common.MapStr{
 						"pattern": "^test",
 					},
+					"json": common.MapStr{
+						"keys_under_root": "true",
+					},
 				},
 				"metrics": common.MapStr{
 					"module": "prometheus",

diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes_test.go b/libbeat/autodiscover/providers/kubernetes/kubernetes_test.go
@@ -89,18 +89,20 @@ func TestGenerateHints(t *testing.T) {
 		},
 		// Scenarios being tested:
 		// logs/multiline.pattern must be a nested common.MapStr under hints.logs
+		// logs/json.keys_under_root must be a nested common.MapStr under hints.logs
 		// metrics/module must be found in hints.metrics
 		// not.to.include must not be part of hints
 		// period is annotated at both container and pod level. Container level value must be in hints
 		{
 			event: bus.Event{
 				"kubernetes": common.MapStr{
 					"annotations": getNestedAnnotations(common.MapStr{
-						"co.elastic.logs/multiline.pattern": "^test",
-						"co.elastic.metrics/module":         "prometheus",
-						"co.elastic.metrics/period":         "10s",
-						"co.elastic.metrics.foobar/period":  "15s",
-						"not.to.include":                    "true",
+						"co.elastic.logs/multiline.pattern":    "^test",
+						"co.elastic.logs/json.keys_under_root": "true",
+						"co.elastic.metrics/module":            "prometheus",
+						"co.elastic.metrics/period":            "10s",
+						"co.elastic.metrics.foobar/period":     "15s",
+						"not.to.include":                       "true",
 					}),
 					"container": common.MapStr{
 						"name":    "foobar",
@@ -112,11 +114,12 @@ func TestGenerateHints(t *testing.T) {
 			result: bus.Event{
 				"kubernetes": common.MapStr{
 					"annotations": getNestedAnnotations(common.MapStr{
-						"co.elastic.logs/multiline.pattern": "^test",
-						"co.elastic.metrics/module":         "prometheus",
-						"not.to.include":                    "true",
-						"co.elastic.metrics/period":         "10s",
-						"co.elastic.metrics.foobar/period":  "15s",
+						"co.elastic.logs/multiline.pattern":    "^test",
+						"co.elastic.logs/json.keys_under_root": "true",
+						"co.elastic.metrics/module":            "prometheus",
+						"not.to.include":                       "true",
+						"co.elastic.metrics/period":            "10s",
+						"co.elastic.metrics.foobar/period":     "15s",
 					}),
 					"container": common.MapStr{
 						"name":    "foobar",
@@ -129,6 +132,9 @@ func TestGenerateHints(t *testing.T) {
 						"multiline": common.MapStr{
 							"pattern": "^test",
 						},
+						"json": common.MapStr{
+							"keys_under_root": "true",
+						},
 					},
 					"metrics": common.MapStr{
 						"module": "prometheus",