[Filebeat] Improve ECS categorization field mappings for mongodb module

CHANGELOG.next.asciidoc

@@ -215,6 +215,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve AWS cloudtrail field mappings {issue}16086[16086] {issue}16110[16110] {pull}17155[17155]
 - Added documentation for running Filebeat in Cloud Foundry. {pull}17275[17275]
 - Move azure-eventhub input to GA. {issue}15671[15671] {pull}17313[17313]
+- Improve ECS categorization field mappings in mongodb module. {issue}16170[16170] {pull}17371[17371]
 
 *Heartbeat*
 

filebeat/module/mongodb/log/ingest/pipeline.yml

@@ -0,0 +1,43 @@
+description: Pipeline for parsing MongoDB logs
+processors:
+- grok:
+    field: message
+    patterns:
+    - '%{TIMESTAMP_ISO8601:mongodb.log.timestamp}%{SPACE}%{MONGO3_SEVERITY:log.level}%{SPACE}%{MONGO3_COMPONENT:mongodb.log.component}%{SPACE}(?:\[%{DATA:mongodb.log.context}\])?%{SPACE}%{GREEDYDATA:message}'
+    ignore_missing: true
+- rename:
+    field: '@timestamp'
+    target_field: event.created
+- date:
+    field: mongodb.log.timestamp
+    target_field: '@timestamp'
+    formats:
+    - yyyy-MM-dd'T'HH:mm:ss.SSSZZ
+- remove:
+    field: mongodb.log.timestamp
+- set:
+    field: event.kind
+    value: event
+- append:
+    field: event.category
+    value: database
+- append:
+    field: event.type
+    value: access
+    if: "ctx?.mongodb?.log?.component == 'ACCESS'"
+- append:
+    field: event.type
+    value: change
+    if: "ctx?.mongodb?.log?.component == 'WRITE'"
+- append:
+    field: event.type
+    value: info
+    if: "ctx?.mongodb?.log?.component != 'WRITE' && ctx?.mongodb?.log?.component != 'ACCESS'"
+- append:
+    field: event.type
+    value: error
+    if: "ctx?.log?.level == 'F' || ctx?.log?.level == 'E'"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/mongodb/log/manifest.yml

@@ -7,5 +7,5 @@ var:
     os.windows:
       - c:\data\log\mongod.log
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/log.yml

filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log

@@ -33,3 +33,5 @@
 2018-02-05T14:49:45.605+0100 I FTDC     [signalProcessingThread] Shutting down full-time diagnostic data capture
 2018-02-05T14:49:45.606+0100 I NETWORK  [signalProcessingThread] closing listening socket: 6
 2019-03-07T15:10:26.960+0000 I ASIO [NetworkInterfaceASIO-Replication-0] Successfully connected to dbbox7:27017, took 10ms (1 connections now open to dbbox7:27017)
+2020-03-31T21:19:46.942+0000 E WRITE  [initandlisten] ** ERROR: A write operation resulted in an error. E11000 duplicate key error index: test.people.$_id_ dup key: { : 0 }
+2020-03-31T21:19:47.420+0000 E NETWORK  [initandlisten] ** ERROR: No connection could be made because the target machine actively refused it 127.0.0.1:27017 at System.Net.Sockets.Socket.EndConnect