[Filebeat] Improve ECS categorization field mappings in system module (…

…elastic#18065)

* Improve ECS categorization field mappings in system module

- auth
  + event.kind
  + event.category (make array)
  + event.type (make array)
  + capture useradd, usermod, userdel
  + capture groupadd, groupmod, groupdel
  + related.ip
  + related.user
- syslog
  + event.kind

Closes elastic#16031

CHANGELOG.next.asciidoc

@@ -287,6 +287,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]
 - Improve ECS categorization field mappings for zeek module. {issue}16029[16029] {pull}17738[17738]
 - Improve ECS categorization field mappings for netflow module. {issue}16135[16135] {pull}18108[18108]
+- Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065]
 
 *Heartbeat*
 

filebeat/module/system/auth/ingest/pipeline.yml

@@ -0,0 +1,145 @@
+description: Pipeline for parsing system authorisation/secure logs
+processors:
+- grok:
+    field: message
+    ignore_missing: true
+    pattern_definitions:
+      GREEDYMULTILINE: |-
+        (.|
+        )*
+      TIMESTAMP: (?:%{TIMESTAMP_ISO8601}|%{SYSLOGTIMESTAMP})
+    patterns:
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      %{DATA:system.auth.ssh.event} %{DATA:system.auth.ssh.method} for (invalid user
+      )?%{DATA:user.name} from %{IPORHOST:source.ip} port %{NUMBER:source.port:long}
+      ssh2(: %{GREEDYDATA:system.auth.ssh.signature})?'
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      %{DATA:system.auth.ssh.event} user %{DATA:user.name} from %{IPORHOST:source.ip}'
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}'
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      \s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty}
+      ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}'
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}'
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id},
+      home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$'
+    - '%{TIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\[%{POSINT:process.pid:long}\])?:
+      %{GREEDYMULTILINE:system.auth.message}'
+- remove:
+    field: message
+- rename:
+    field: system.auth.message
+    target_field: message
+    ignore_missing: true
+- set:
+    field: source.ip
+    value: '{{system.auth.ssh.dropped_ip}}'
+    if: "ctx?.system?.auth?.ssh?.dropped_ip != null"
+- date:
+    if: ctx.event.timezone == null
+    field: system.auth.timestamp
+    target_field: '@timestamp'
+    formats:
+    - MMM  d HH:mm:ss
+    - MMM dd HH:mm:ss
+    - ISO8601
+    on_failure:
+    - append:
+        field: error.message
+        value: '{{ _ingest.on_failure_message }}'
+- date:
+    if: ctx.event.timezone != null
+    field: system.auth.timestamp
+    target_field: '@timestamp'
+    formats:
+    - MMM  d HH:mm:ss
+    - MMM dd HH:mm:ss
+    - ISO8601
+    timezone: '{{ event.timezone }}'
+    on_failure:
+    - append:
+        field: error.message
+        value: '{{ _ingest.on_failure_message }}'
+- remove:
+    field: system.auth.timestamp
+- geoip:
+    field: source.ip
+    target_field: source.geo
+    ignore_failure: true
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
+- set:
+    field: event.kind
+    value: event
+- script:
+    lang: painless
+    ignore_failure: true
+    source: >-
+      if (ctx.system.auth.ssh.event == "Accepted") {
+        ctx.event.type = ["authentication_success", "info"];
+        ctx.event.category = ["authentication"];
+        ctx.event.action = "ssh_login";
+        ctx.event.outcome = "success";
+      } else if (ctx.system.auth.ssh.event == "Invalid" || ctx.system.auth.ssh.event == "Failed") {
+        ctx.event.type = ["authentication_failure", "info"];
+        ctx.event.category = ["authentication"];
+        ctx.event.action = "ssh_login";
+        ctx.event.outcome = "failure";
+      }
+
+- append:
+    field: event.category
+    value: iam
+    if: "ctx?.process?.name != null && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name)"
+- set:
+    field: event.outcome
+    value: success
+    if: "ctx?.process?.name != null && ['groupadd', 'groupdel', 'groupmod', 'useradd', 'userdel', 'usermod'].contains(ctx.process.name)"
+- append:
+    field: event.type
+    value: user
+    if: "ctx?.process?.name != null && ['useradd', 'userdel', 'usermod'].contains(ctx.process.name)"
+- append:
+    field: event.type
+    value: group
+    if: "ctx?.process?.name != null && ['groupadd', 'groupdel', 'groupmod'].contains(ctx.process.name)"
+- append:
+    field: event.type
+    value: creation
+    if: "ctx?.process?.name != null && ['useradd', 'groupadd'].contains(ctx.process.name)"
+- append:
+    field: event.type
+    value: deletion
+    if: "ctx?.process?.name != null && ['userdel', 'groupdel'].contains(ctx.process.name)"
+- append:
+    field: event.type
+    value: change
+    if: "ctx?.process?.name != null && ['usermod', 'groupmod'].contains(ctx.process.name)"
+- append:
+    field: related.user
+    value: "{{user.name}}"
+    if: "ctx?.user?.name != null"
+- append:
+    field: related.ip
+    value: "{{source.ip}}"
+    if: "ctx?.source?.ip != null"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

filebeat/module/system/auth/manifest.yml

@@ -11,5 +11,5 @@ var:
       - /var/log/secure.log*
     os.windows: []
 
-ingest_pipeline: ingest/pipeline.json
+ingest_pipeline: ingest/pipeline.yml
 input: config/auth.yml