From a1975e212ac66dc62978943a564d5c794d1e9851 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Tue, 31 Mar 2020 08:56:59 -0500 Subject: [PATCH 1/2] convert mongodb pipeline to yaml --- .../module/mongodb/log/ingest/pipeline.json | 36 ------------------- .../module/mongodb/log/ingest/pipeline.yml | 21 +++++++++++ filebeat/module/mongodb/log/manifest.yml | 2 +- 3 files changed, 22 insertions(+), 37 deletions(-) delete mode 100755 filebeat/module/mongodb/log/ingest/pipeline.json create mode 100644 filebeat/module/mongodb/log/ingest/pipeline.yml diff --git a/filebeat/module/mongodb/log/ingest/pipeline.json b/filebeat/module/mongodb/log/ingest/pipeline.json deleted file mode 100755 index 7af8a8aeb0a..00000000000 --- a/filebeat/module/mongodb/log/ingest/pipeline.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "description": "Pipeline for parsing MongoDB logs", - "processors": [{ - "grok": { - "field": "message", - "patterns":[ - "%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\\[%{DATA:mongodb.log.context}\\])?%{SPACE}%{GREEDYDATA:message}" - ], - "ignore_missing": true - } - }, - { - "rename": { - "field": "@timestamp", - "target_field": "event.created" - } - }, - { - "date": { - "field": "mongodb.log.timestamp", - "target_field": "@timestamp", - "formats": ["yyyy-MM-dd'T'HH:mm:ss.SSSZZ"] - } - }, - { - "remove": { - "field": "mongodb.log.timestamp" - } - }], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] -} diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml new file mode 100644 index 00000000000..0eeeb8f2bb9 --- /dev/null +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -0,0 +1,21 @@ +description: Pipeline for parsing MongoDB logs +processors: +- grok: + field: message + patterns: + - '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}' + ignore_missing: true +- rename: + field: '@timestamp' + target_field: event.created +- date: + field: mongodb.log.timestamp + target_field: '@timestamp' + formats: + - yyyy-MM-dd'T'HH:mm:ss.SSSZZ +- remove: + field: mongodb.log.timestamp +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/filebeat/module/mongodb/log/manifest.yml b/filebeat/module/mongodb/log/manifest.yml index f77eaa89cb9..0eab78a0548 100644 --- a/filebeat/module/mongodb/log/manifest.yml +++ b/filebeat/module/mongodb/log/manifest.yml @@ -7,5 +7,5 @@ var: os.windows: - c:\data\log\mongod.log -ingest_pipeline: ingest/pipeline.json +ingest_pipeline: ingest/pipeline.yml input: config/log.yml From 59f9ffd46f64f226eef1e75ad9a0afae8d5e47d7 Mon Sep 17 00:00:00 2001 From: "Lee E. Hinman" Date: Tue, 31 Mar 2020 13:42:30 -0500 Subject: [PATCH 2/2] Improve ECS categorization field mapping for mongodb module - event.kind - event.category - event.type Closes #16170 --- CHANGELOG.next.asciidoc | 1 + .../module/mongodb/log/ingest/pipeline.yml | 22 ++ .../log/test/mongodb-debian-3.2.11.log | 2 + .../mongodb-debian-3.2.11.log-expected.json | 287 ++++++++++++++++++ 4 files changed, 312 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 36a2ef96432..f01feb964ae 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -215,6 +215,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155] - Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275] - Move azure-eventhub input to GA. {issue}15671[15671] {pull}17313[17313] +- Improve ECS categorization field mappings in mongodb module. {issue}16170[16170] {pull}17371[17371] *Heartbeat* diff --git a/filebeat/module/mongodb/log/ingest/pipeline.yml b/filebeat/module/mongodb/log/ingest/pipeline.yml index 0eeeb8f2bb9..6460a2b02c6 100644 --- a/filebeat/module/mongodb/log/ingest/pipeline.yml +++ b/filebeat/module/mongodb/log/ingest/pipeline.yml @@ -15,6 +15,28 @@ processors: - yyyy-MM-dd'T'HH:mm:ss.SSSZZ - remove: field: mongodb.log.timestamp +- set: + field: event.kind + value: event +- append: + field: event.category + value: database +- append: + field: event.type + value: access + if: "ctx?.mongodb?.log?.component == 'ACCESS'" +- append: + field: event.type + value: change + if: "ctx?.mongodb?.log?.component == 'WRITE'" +- append: + field: event.type + value: info + if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'" +- append: + field: event.type + value: error + if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'" on_failure: - set: field: error.message diff --git a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log index 0960db32921..92aee00f293 100644 --- a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log +++ b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log @@ -33,3 +33,5 @@ 2018-02-05T14:49:45.605+0100 I FTDC [signalProcessingThread] Shutting down full-time diagnostic data capture 2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] closing listening socket: 6 2019-03-07T15:10:26.960+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Successfully connected to dbbox7:27017, took 10ms (1 connections now open to dbbox7:27017) +2020-03-31T21:19:46.942+0000 E WRITE [initandlisten] ** ERROR: A write operation resulted in an error. E11000 duplicate key error index: test.people.$_id_ dup key: { : 0 } +2020-03-31T21:19:47.420+0000 E NETWORK [initandlisten] ** ERROR: No connection could be made because the target machine actively refused it 127.0.0.1:27017 at System.Net.Sockets.Socket.EndConnect diff --git a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json index c7a4a01e3cc..c0337fac9c7 100644 --- a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json +++ b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json @@ -1,8 +1,15 @@ [ { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -14,8 +21,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -27,8 +41,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -40,8 +61,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.677Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -53,8 +81,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.724Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -66,8 +101,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.724Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -79,8 +121,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.744Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -92,8 +141,15 @@ }, { "@timestamp": "2018-02-05T12:50:55.170Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -105,8 +161,15 @@ }, { "@timestamp": "2018-02-05T12:50:55.487Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -118,8 +181,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -131,8 +201,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -144,8 +221,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -157,8 +241,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -170,8 +261,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -183,8 +281,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.688Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -196,8 +301,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -209,8 +321,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -222,8 +341,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -235,8 +361,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -248,8 +381,15 @@ }, { "@timestamp": "2018-02-05T12:50:55.170Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -261,8 +401,15 @@ }, { "@timestamp": "2018-02-05T12:50:56.180Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -274,8 +421,15 @@ }, { "@timestamp": "2018-02-05T13:15:42.095Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -287,8 +441,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -300,8 +461,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -313,8 +481,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.688Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -326,8 +501,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -339,8 +521,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -352,8 +541,15 @@ }, { "@timestamp": "2018-02-05T12:44:56.657Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -365,8 +561,15 @@ }, { "@timestamp": "2018-02-05T12:50:55.487Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -378,8 +581,15 @@ }, { "@timestamp": "2018-02-05T12:50:56.180Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -391,8 +601,15 @@ }, { "@timestamp": "2018-02-05T13:11:41.401Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -404,8 +621,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.605Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -417,8 +641,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.605Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -430,8 +661,15 @@ }, { "@timestamp": "2018-02-05T13:49:45.606Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -443,8 +681,15 @@ }, { "@timestamp": "2019-03-07T15:10:26.960Z", + "event.category": [ + "database" + ], "event.dataset": "mongodb.log", + "event.kind": "event", "event.module": "mongodb", + "event.type": [ + "info" + ], "fileset.name": "log", "input.type": "log", "log.level": "I", @@ -453,5 +698,47 @@ "mongodb.log.component": "ASIO", "mongodb.log.context": "NetworkInterfaceASIO-Replication-0", "service.type": "mongodb" + }, + { + "@timestamp": "2020-03-31T21:19:46.942Z", + "event.category": [ + "database" + ], + "event.dataset": "mongodb.log", + "event.kind": "event", + "event.module": "mongodb", + "event.type": [ + "change", + "error" + ], + "fileset.name": "log", + "input.type": "log", + "log.level": "E", + "log.offset": 4132, + "message": "** ERROR: A write operation resulted in an error. E11000 duplicate key error index: test.people.$_id_ dup key: { : 0 }", + "mongodb.log.component": "WRITE", + "mongodb.log.context": "initandlisten", + "service.type": "mongodb" + }, + { + "@timestamp": "2020-03-31T21:19:47.420Z", + "event.category": [ + "database" + ], + "event.dataset": "mongodb.log", + "event.kind": "event", + "event.module": "mongodb", + "event.type": [ + "info", + "error" + ], + "fileset.name": "log", + "input.type": "log", + "log.level": "E", + "log.offset": 4305, + "message": "** ERROR: No connection could be made because the target machine actively refused it 127.0.0.1:27017 at System.Net.Sockets.Socket.EndConnect", + "mongodb.log.component": "NETWORK", + "mongodb.log.context": "initandlisten", + "service.type": "mongodb" } ] \ No newline at end of file