From c3aedc89522479228a9250e29fd6f8b5bb80ad0d Mon Sep 17 00:00:00 2001 From: ruflin Date: Tue, 2 May 2017 08:49:46 +0200 Subject: [PATCH] Move json_error to error.message and error.type All errors are now under the error namespace. This should also be true for json errors. Currently they are put under `json_error`. This PR changes it to `error.message` and `error.type: json`. This makes it easy to search for all json errors. This is a breaking change. * Docs updated * Tests updated The naming of the config option was not touched for better backward compatibility. --- CHANGELOG.asciidoc | 3 ++- filebeat/_meta/common.full.p2.yml | 2 +- .../configuration/filebeat-options.asciidoc | 2 +- filebeat/filebeat.full.yml | 2 +- filebeat/harvester/json_test.go | 16 ++++++++-------- filebeat/harvester/log.go | 2 +- filebeat/harvester/reader/json.go | 14 +++++++------- filebeat/harvester/reader/json_test.go | 8 ++++---- filebeat/tests/system/test_json.py | 12 ++++++------ libbeat/common/jsontransform/jsonhelper.go | 15 ++++++++++----- libbeat/processors/actions/decode_json_fields.go | 2 +- 11 files changed, 42 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 41b49ea66ec..776f6ca3bc4 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -84,7 +84,7 @@ https://github.com/elastic/beats/compare/v5.4.0...v6.0.0-alpha1[View commits] *Affecting all Beats* -- Introduce beat version in the Elasticsearch index and mapping template {pull}3527[3527] +- Introduce beat version in the Elasticsearch index and mapping template {pull}3527[3527] - Usage of field `_type` is now ignored and hardcoded to `doc`. {pull}3757[3757] - Change vendor manager from glide to govendor. {pull}3851[3851] - Rename `error` field to `error.message`. {pull}3987[3987] @@ -97,6 +97,7 @@ https://github.com/elastic/beats/compare/v5.4.0...v6.0.0-alpha1[View commits] - Remove deprecated config options force_close_files and close_older. {pull}3768[3768] - Change clean_removed behaviour to also remove states for files which cannot be found anymore under the same name. {pull}3827[3827] - Remove `document_type` config option. Use `fields` instead. {pull}4204[4204] +- Move json_error under error.message and error.key. {pull}4167[4167] *Packetbeat* diff --git a/filebeat/_meta/common.full.p2.yml b/filebeat/_meta/common.full.p2.yml index 61d34414c9d..1a7b64c4658 100644 --- a/filebeat/_meta/common.full.p2.yml +++ b/filebeat/_meta/common.full.p2.yml @@ -102,7 +102,7 @@ filebeat.prospectors: # in case of conflicts. #json.overwrite_keys: false - # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON + # If this setting is enabled, Filebeat adds a "error.message" and "error.key: json" key in case of JSON # unmarshaling errors or when a text key is defined in the configuration but cannot # be used. #json.add_error_key: false diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index 70836c1bbf4..b3c823cffea 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -335,7 +335,7 @@ If you enable this setting, the keys are copied top level in the output document *`overwrite_keys`*:: If `keys_under_root` and this setting are enabled, then the values from the decoded JSON object overwrite the fields that Filebeat normally adds (type, source, offset, etc.) in case of conflicts. -*`add_error_key`*:: If this setting is enabled, Filebeat adds a "json_error" key in case of JSON +*`add_error_key`*:: If this setting is enabled, Filebeat adds a "error.message" and "error.type: json" key in case of JSON unmarshalling errors or when a `message_key` is defined in the configuration but cannot be used. *`message_key`*:: An optional configuration setting that specifies a JSON key on diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 5b107cacfd1..06036fc779a 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -271,7 +271,7 @@ filebeat.prospectors: # in case of conflicts. #json.overwrite_keys: false - # If this setting is enabled, Filebeat adds a "json_error" key in case of JSON + # If this setting is enabled, Filebeat adds a "error.message" and "error.key: json" key in case of JSON # unmarshaling errors or when a text key is defined in the configuration but cannot # be used. #json.add_error_key: false diff --git a/filebeat/harvester/json_test.go b/filebeat/harvester/json_test.go index 849e96376e2..19026b0b0fd 100644 --- a/filebeat/harvester/json_test.go +++ b/filebeat/harvester/json_test.go @@ -81,7 +81,7 @@ func TestAddJSONFields(t *testing.T) { ExpectedItems: common.MapStr{ "@timestamp": common.Time(now), "type": "test", - "json_error": "@timestamp not overwritten (parse error on 2016-04-05T18:47:18.44XX4Z)", + "error": common.MapStr{"type": "json", "message": "@timestamp not overwritten (parse error on 2016-04-05T18:47:18.44XX4Z)"}, }, }, { @@ -93,7 +93,7 @@ func TestAddJSONFields(t *testing.T) { ExpectedItems: common.MapStr{ "@timestamp": common.Time(now), "type": "test", - "json_error": "@timestamp not overwritten (not string)", + "error": common.MapStr{"type": "json", "message": "@timestamp not overwritten (not string)"}, }, }, { @@ -102,8 +102,8 @@ func TestAddJSONFields(t *testing.T) { Text: &text, JSONConfig: reader.JSONConfig{KeysUnderRoot: true, OverwriteKeys: true}, ExpectedItems: common.MapStr{ - "type": "test_type", - "json_error": "type not overwritten (not string)", + "type": "test_type", + "error": common.MapStr{"type": "json", "message": "type not overwritten (not string)"}, }, }, { @@ -112,8 +112,8 @@ func TestAddJSONFields(t *testing.T) { Text: &text, JSONConfig: reader.JSONConfig{KeysUnderRoot: true, OverwriteKeys: true}, ExpectedItems: common.MapStr{ - "type": "test_type", - "json_error": "type not overwritten (invalid value [])", + "type": "test_type", + "error": common.MapStr{"type": "json", "message": "type not overwritten (invalid value [])"}, }, }, { @@ -122,8 +122,8 @@ func TestAddJSONFields(t *testing.T) { Text: &text, JSONConfig: reader.JSONConfig{KeysUnderRoot: true, OverwriteKeys: true}, ExpectedItems: common.MapStr{ - "type": "test_type", - "json_error": "type not overwritten (invalid value [_type])", + "type": "test_type", + "error": common.MapStr{"type": "json", "message": "type not overwritten (invalid value [_type])"}, }, }, } diff --git a/filebeat/harvester/log.go b/filebeat/harvester/log.go index 1aabc3ab2b6..6e115effcaa 100644 --- a/filebeat/harvester/log.go +++ b/filebeat/harvester/log.go @@ -424,6 +424,6 @@ func (h *Harvester) mergeJSONFields(data common.MapStr, jsonFields common.MapStr // Delete existing json key delete(data, "json") - jsontransform.WriteJSONKeys(data, jsonFields, h.config.JSON.OverwriteKeys, reader.JsonErrorKey) + jsontransform.WriteJSONKeys(data, jsonFields, h.config.JSON.OverwriteKeys) } } diff --git a/filebeat/harvester/reader/json.go b/filebeat/harvester/reader/json.go index 5da01eb30ec..1f9ce254f93 100644 --- a/filebeat/harvester/reader/json.go +++ b/filebeat/harvester/reader/json.go @@ -10,10 +10,6 @@ import ( "github.com/elastic/beats/libbeat/logp" ) -const ( - JsonErrorKey = "json_error" -) - type JSON struct { reader Reader cfg *JSONConfig @@ -33,7 +29,7 @@ func (r *JSON) decodeJSON(text []byte) ([]byte, common.MapStr) { if err != nil || jsonFields == nil { logp.Err("Error decoding JSON: %v", err) if r.cfg.AddErrorKey { - jsonFields = common.MapStr{JsonErrorKey: fmt.Sprintf("Error decoding JSON: %v", err)} + jsonFields = common.MapStr{"error": createJSONError(fmt.Sprintf("Error decoding JSON: %v", err))} } return text, jsonFields } @@ -45,7 +41,7 @@ func (r *JSON) decodeJSON(text []byte) ([]byte, common.MapStr) { textValue, ok := jsonFields[r.cfg.MessageKey] if !ok { if r.cfg.AddErrorKey { - jsonFields[JsonErrorKey] = fmt.Sprintf("Key '%s' not found", r.cfg.MessageKey) + jsonFields["error"] = createJSONError(fmt.Sprintf("Key '%s' not found", r.cfg.MessageKey)) } return []byte(""), jsonFields } @@ -53,7 +49,7 @@ func (r *JSON) decodeJSON(text []byte) ([]byte, common.MapStr) { textString, ok := textValue.(string) if !ok { if r.cfg.AddErrorKey { - jsonFields[JsonErrorKey] = fmt.Sprintf("Value of key '%s' is not a string", r.cfg.MessageKey) + jsonFields["error"] = createJSONError(fmt.Sprintf("Value of key '%s' is not a string", r.cfg.MessageKey)) } return []byte(""), jsonFields } @@ -86,3 +82,7 @@ func (r *JSON) Next() (Message, error) { message.AddFields(common.MapStr{"json": fields}) return message, nil } + +func createJSONError(message string) common.MapStr { + return common.MapStr{"message": message, "type": "json"} +} diff --git a/filebeat/harvester/reader/json_test.go b/filebeat/harvester/reader/json_test.go index bc5d1437677..f27f8f3baca 100644 --- a/filebeat/harvester/reader/json_test.go +++ b/filebeat/harvester/reader/json_test.go @@ -121,28 +121,28 @@ func TestDecodeJSON(t *testing.T) { Text: `null`, Config: JSONConfig{MessageKey: "value", AddErrorKey: true}, ExpectedText: `null`, - ExpectedMap: common.MapStr{"json_error": "Error decoding JSON: "}, + ExpectedMap: common.MapStr{"error": common.MapStr{"message": "Error decoding JSON: ", "type": "json"}}, }, { // Add key error helps debugging this Text: `{"message": "test", "value": "`, Config: JSONConfig{MessageKey: "value", AddErrorKey: true}, ExpectedText: `{"message": "test", "value": "`, - ExpectedMap: common.MapStr{"json_error": "Error decoding JSON: unexpected EOF"}, + ExpectedMap: common.MapStr{"error": common.MapStr{"message": "Error decoding JSON: unexpected EOF", "type": "json"}}, }, { // If the text key is not found, put an error Text: `{"message": "test", "value": "1"}`, Config: JSONConfig{MessageKey: "hello", AddErrorKey: true}, ExpectedText: ``, - ExpectedMap: common.MapStr{"message": "test", "value": "1", "json_error": "Key 'hello' not found"}, + ExpectedMap: common.MapStr{"message": "test", "value": "1", "error": common.MapStr{"message": "Key 'hello' not found", "type": "json"}}, }, { // If the text key is found, but not a string, put an error Text: `{"message": "test", "value": 1}`, Config: JSONConfig{MessageKey: "value", AddErrorKey: true}, ExpectedText: ``, - ExpectedMap: common.MapStr{"message": "test", "value": int64(1), "json_error": "Value of key 'value' is not a string"}, + ExpectedMap: common.MapStr{"message": "test", "value": int64(1), "error": common.MapStr{"message": "Value of key 'value' is not a string", "type": "json"}}, }, { // Without a text key, simple return the json and an empty text diff --git a/filebeat/tests/system/test_json.py b/filebeat/tests/system/test_json.py index 297055e3f00..ec1df37db1d 100644 --- a/filebeat/tests/system/test_json.py +++ b/filebeat/tests/system/test_json.py @@ -199,16 +199,16 @@ def test_timestamp_in_message(self): assert output[0]["@timestamp"] == "2016-04-05T18:47:18.444Z" assert output[1]["@timestamp"] != "invalid" - assert output[1]["json_error"] == \ + assert output[1]["error.message"] == \ "@timestamp not overwritten (parse error on invalid)" - assert output[2]["json_error"] == \ + assert output[2]["error.message"] == \ "@timestamp not overwritten (not string)" - assert "json_error" not in output[3] + assert "error" not in output[3] assert output[3]["@timestamp"] == "2016-04-05T18:47:18.444Z", output[3]["@timestamp"] - assert "json_error" not in output[4] + assert "error" not in output[4] assert output[4]["@timestamp"] == "2016-04-05T18:47:18.000Z", output[4]["@timestamp"] def test_type_in_message(self): @@ -241,11 +241,11 @@ def test_type_in_message(self): assert output[0]["type"] == "test" assert "type" not in output[1] - assert output[1]["json_error"] == \ + assert output[1]["error.message"] == \ "type not overwritten (not string)" assert "type" not in output[2] - assert output[2]["json_error"] == \ + assert output[2]["error.message"] == \ "type not overwritten (not string)" def test_with_generic_filtering(self): diff --git a/libbeat/common/jsontransform/jsonhelper.go b/libbeat/common/jsontransform/jsonhelper.go index 37bdb7e93e8..1691ce3cc47 100644 --- a/libbeat/common/jsontransform/jsonhelper.go +++ b/libbeat/common/jsontransform/jsonhelper.go @@ -8,14 +8,15 @@ import ( "github.com/elastic/beats/libbeat/logp" ) -func WriteJSONKeys(event common.MapStr, keys map[string]interface{}, overwriteKeys bool, errorKey string) { +// WriteJSONKeys writes the json keys to the given event based on the overwriteKeys option +func WriteJSONKeys(event common.MapStr, keys map[string]interface{}, overwriteKeys bool) { for k, v := range keys { if overwriteKeys { if k == "@timestamp" { vstr, ok := v.(string) if !ok { logp.Err("JSON: Won't overwrite @timestamp because value is not string") - event[errorKey] = "@timestamp not overwritten (not string)" + event["error"] = createJSONError("@timestamp not overwritten (not string)") continue } @@ -23,7 +24,7 @@ func WriteJSONKeys(event common.MapStr, keys map[string]interface{}, overwriteKe ts, err := time.Parse(time.RFC3339, vstr) if err != nil { logp.Err("JSON: Won't overwrite @timestamp because of parsing error: %v", err) - event[errorKey] = fmt.Sprintf("@timestamp not overwritten (parse error on %s)", vstr) + event["error"] = createJSONError(fmt.Sprintf("@timestamp not overwritten (parse error on %s)", vstr)) continue } event[k] = common.Time(ts) @@ -31,12 +32,12 @@ func WriteJSONKeys(event common.MapStr, keys map[string]interface{}, overwriteKe vstr, ok := v.(string) if !ok { logp.Err("JSON: Won't overwrite type because value is not string") - event[errorKey] = "type not overwritten (not string)" + event["error"] = createJSONError("type not overwritten (not string)") continue } if len(vstr) == 0 || vstr[0] == '_' { logp.Err("JSON: Won't overwrite type because value is empty or starts with an underscore") - event[errorKey] = fmt.Sprintf("type not overwritten (invalid value [%s])", vstr) + event["error"] = createJSONError(fmt.Sprintf("type not overwritten (invalid value [%s])", vstr)) continue } event[k] = vstr @@ -49,3 +50,7 @@ func WriteJSONKeys(event common.MapStr, keys map[string]interface{}, overwriteKe } } + +func createJSONError(message string) common.MapStr { + return common.MapStr{"message": message, "type": "json"} +} diff --git a/libbeat/processors/actions/decode_json_fields.go b/libbeat/processors/actions/decode_json_fields.go index b34b7fe807a..c1e6778bc4b 100644 --- a/libbeat/processors/actions/decode_json_fields.go +++ b/libbeat/processors/actions/decode_json_fields.go @@ -87,7 +87,7 @@ func (f decodeJSONFields) Run(event common.MapStr) (common.MapStr, error) { default: errs = append(errs, errors.New("Error trying to add target to root.").Error()) case map[string]interface{}: - jsontransform.WriteJSONKeys(event, t, f.overwriteKeys, "json_error") + jsontransform.WriteJSONKeys(event, t, f.overwriteKeys) } } } else {