Allow users to convert timezone in elasticsearch module filesets (ela…

…stic#9761) This PR updates the following filesets in the `elasticsearch` Filebeat module to accept a `var.convert_timezone` configuration setting: * [x] server * [x] audit * [x] slowlog * [x] deprecation The `gc` fileset was not modified because Elasticsearch GC logs already emit timestamps with timezone information in them. Fixes partially elastic#9756. Related: elastic#9797. (cherry picked from commit 3559e58)
ycombinator · Jan 30, 2019 · f87f2ef · f87f2ef
1 parent 9f4b205
commit f87f2ef
Show file tree

Hide file tree

Showing 22 changed files with 647 additions and 504 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -38,6 +38,9 @@ https://github.com/elastic/beats/compare/1035569addc4a3b29ffa14f8a08c27c1ace16ef
 
 - Fix bad bytes count in `docker` input when filtering by stream. {pull}10211[10211]
 - Add `convert_timezone` option to Logstash module to convert dates to UTC. {issue}9756[9756] {pull}9797[9797]
+- Add `convert_timezone` option to Elasticsearch module to convert dates to UTC. {issue}9756[9756] {pull}9761[9761]
+
+*Heartbeat*
 
 *Heartbeat*
 

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -92,6 +92,9 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   gc:
     enabled: true
     # Set custom paths for the log files. If left empty,
@@ -104,19 +107,29 @@ filebeat.modules:
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   slowlog:
     enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   deprecation:
     enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
-#------------------------------- haproxy Module ------------------------------
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
+
+#------------------------------- Haproxy Module -------------------------------
 - module: haproxy
   # All logs
   log:

diff --git a/filebeat/module/elasticsearch/_meta/config.yml b/filebeat/module/elasticsearch/_meta/config.yml
@@ -7,6 +7,9 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   gc:
     enabled: true
     # Set custom paths for the log files. If left empty,
@@ -19,14 +22,24 @@
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   slowlog:
     enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
 
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
   deprecation:
     enabled: true
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.
     #var.paths:
+
+    # Convert the timestamp to UTC. Requires Elasticsearch >= 6.1.
+    #var.convert_timezone: false
+
diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml
@@ -5,6 +5,11 @@ paths:
 {{ end }}
 exclude_files: [".gz$"]
 
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
+
 fields:
   service.name: "elasticsearch"
 fields_under_root: true
diff --git a/filebeat/module/elasticsearch/audit/ingest/pipeline.json b/filebeat/module/elasticsearch/audit/ingest/pipeline.json
@@ -16,9 +16,19 @@
           }
       },
       {
-          "rename": {
+          "date": {
               "field": "elasticsearch.audit.timestamp",
-              "target_field": "@timestamp"
+              "target_field": "@timestamp",
+              "formats": [
+                "ISO8601"
+            ],
+            {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+            "ignore_failure": true
+          }
+      },
+      {
+        "remove": {
+            "field": "elasticsearch.audit.timestamp"
           }
       }
     ],

diff --git a/filebeat/module/elasticsearch/audit/manifest.yml b/filebeat/module/elasticsearch/audit/manifest.yml
@@ -8,6 +8,13 @@ var:
       - /usr/local/var/lib/elasticsearch/*_access.log
     os.windows:
       - c:/ProgramData/Elastic/Elasticsearch/logs/*_access.log
+  - name: convert_timezone
+    default: false
+    # if ES < 6.1.0, this flag switches to false automatically when evaluating the
+    # pipeline
+    min_elasticsearch_version:
+      version: 6.1.0
+      value: false
 
 ingest_pipeline: ingest/pipeline.json
 input: config/audit.yml
diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json
@@ -1,121 +1,121 @@
 [
     {
-        "@timestamp": "2018-06-19T05:16:15,549", 
-        "elasticsearch.audit.event_type": "authentication_failed", 
-        "elasticsearch.audit.layer": "rest", 
-        "elasticsearch.audit.origin_address": "147.107.128.77", 
-        "elasticsearch.audit.principal": "i030648", 
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", 
-        "offset": 0, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:16:15,549Z",
+        "elasticsearch.audit.event_type": "authentication_failed",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.origin_address": "147.107.128.77",
+        "elasticsearch.audit.principal": "i030648",
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed]        origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]",
+        "offset": 0,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
-    }, 
+    },
     {
-        "@timestamp": "2018-06-19T05:07:52,304", 
-        "elasticsearch.audit.event_type": "authentication_failed", 
-        "elasticsearch.audit.layer": "rest", 
-        "elasticsearch.audit.origin_address": "172.22.0.3", 
-        "elasticsearch.audit.principal": "rado", 
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
-        "elasticsearch.node.name": "v_VJhjV", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", 
-        "offset": 155, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:07:52,304Z",
+        "elasticsearch.audit.event_type": "authentication_failed",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.origin_address": "172.22.0.3",
+        "elasticsearch.audit.principal": "rado",
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
+        "elasticsearch.node.name": "v_VJhjV",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]",
+        "offset": 155,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
-    }, 
+    },
     {
-        "@timestamp": "2018-06-19T05:00:15,778", 
-        "elasticsearch.audit.action": "indices:data/read/scroll/clear", 
-        "elasticsearch.audit.event_type": "access_granted", 
-        "elasticsearch.audit.layer": "transport", 
-        "elasticsearch.audit.origin_address": "192.168.1.165", 
-        "elasticsearch.audit.origin_type": "local_node", 
-        "elasticsearch.audit.principal": "_xpack_security", 
-        "elasticsearch.audit.request": "ClearScrollRequest", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", 
-        "offset": 306, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:00:15,778Z",
+        "elasticsearch.audit.action": "indices:data/read/scroll/clear",
+        "elasticsearch.audit.event_type": "access_granted",
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin_address": "192.168.1.165",
+        "elasticsearch.audit.origin_type": "local_node",
+        "elasticsearch.audit.principal": "_xpack_security",
+        "elasticsearch.audit.request": "ClearScrollRequest",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:00:15,778] [transport] [access_granted]  origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]",
+        "offset": 306,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
-    }, 
+    },
     {
-        "@timestamp": "2018-06-19T05:07:45,544", 
-        "elasticsearch.audit.event_type": "anonymous_access_denied", 
-        "elasticsearch.audit.layer": "rest", 
-        "elasticsearch.audit.origin_address": "172.22.0.3", 
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
-        "elasticsearch.node.name": "v_VJhjV", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", 
-        "offset": 519, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:07:45,544Z",
+        "elasticsearch.audit.event_type": "anonymous_access_denied",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.origin_address": "172.22.0.3",
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
+        "elasticsearch.node.name": "v_VJhjV",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]",
+        "offset": 519,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
-    }, 
+    },
     {
-        "@timestamp": "2018-06-19T05:26:27,268", 
-        "elasticsearch.audit.event_type": "authentication_failed", 
-        "elasticsearch.audit.layer": "rest", 
-        "elasticsearch.audit.origin_address": "147.107.128.77", 
-        "elasticsearch.audit.principal": "N078801", 
-        "elasticsearch.audit.uri": "/_xpack/security/_authenticate", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", 
-        "offset": 654, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:26:27,268Z",
+        "elasticsearch.audit.event_type": "authentication_failed",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.origin_address": "147.107.128.77",
+        "elasticsearch.audit.principal": "N078801",
+        "elasticsearch.audit.uri": "/_xpack/security/_authenticate",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]",
+        "offset": 654,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
-    }, 
+    },
     {
-        "@timestamp": "2018-06-19T05:55:26,898", 
-        "elasticsearch.audit.action": "cluster:monitor/main", 
-        "elasticsearch.audit.event_type": "access_denied", 
-        "elasticsearch.audit.layer": "transport", 
-        "elasticsearch.audit.origin_address": "147.107.128.77", 
-        "elasticsearch.audit.origin_type": "rest", 
-        "elasticsearch.audit.principal": "_anonymous", 
-        "elasticsearch.audit.request": "MainRequest", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", 
-        "offset": 802, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:55:26,898Z",
+        "elasticsearch.audit.action": "cluster:monitor/main",
+        "elasticsearch.audit.event_type": "access_denied",
+        "elasticsearch.audit.layer": "transport",
+        "elasticsearch.audit.origin_address": "147.107.128.77",
+        "elasticsearch.audit.origin_type": "rest",
+        "elasticsearch.audit.principal": "_anonymous",
+        "elasticsearch.audit.request": "MainRequest",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]",
+        "offset": 802,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
-    }, 
+    },
     {
-        "@timestamp": "2018-06-19T05:24:15,190", 
-        "elasticsearch.audit.event_type": "authentication_failed", 
-        "elasticsearch.audit.layer": "rest", 
-        "elasticsearch.audit.origin_address": "172.18.0.3", 
-        "elasticsearch.audit.principal": "elastic", 
-        "elasticsearch.audit.request_body": "body", 
-        "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip", 
-        "elasticsearch.node.name": "v_VJhjV", 
-        "event.dataset": "elasticsearch.audit", 
-        "fileset.module": "elasticsearch", 
-        "fileset.name": "audit", 
-        "input.type": "log", 
-        "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", 
-        "offset": 986, 
-        "prospector.type": "log", 
+        "@timestamp": "2018-06-19T05:24:15,190Z",
+        "elasticsearch.audit.event_type": "authentication_failed",
+        "elasticsearch.audit.layer": "rest",
+        "elasticsearch.audit.origin_address": "172.18.0.3",
+        "elasticsearch.audit.principal": "elastic",
+        "elasticsearch.audit.request_body": "body",
+        "elasticsearch.audit.uri": "/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip",
+        "elasticsearch.node.name": "v_VJhjV",
+        "event.dataset": "elasticsearch.audit",
+        "fileset.module": "elasticsearch",
+        "fileset.name": "audit",
+        "input.type": "log",
+        "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]",
+        "offset": 986,
+        "prospector.type": "log",
         "service.name": "elasticsearch"
     }
-]
+]
diff --git a/filebeat/module/elasticsearch/deprecation/config/log.yml b/filebeat/module/elasticsearch/deprecation/config/log.yml
@@ -9,6 +9,11 @@ multiline:
   negate: true
   match: after
 
+{{ if .convert_timezone }}
+processors:
+- add_locale: ~
+{{ end }}
+
 fields:
   service.name: "elasticsearch"
 

diff --git a/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json b/filebeat/module/elasticsearch/deprecation/ingest/pipeline.json
@@ -27,9 +27,19 @@
             }
         },
         {
-            "rename": {
+            "date": {
                 "field": "timestamp",
-                "target_field": "@timestamp"
+                "target_field": "@timestamp",
+                "formats": [
+                    "ISO8601"
+                ],
+                {< if .convert_timezone >}"timezone": "{{ event.timezone }}",{< end >}
+                "ignore_failure": true
+            }
+        },
+        {
+            "remove": {
+                "field": "timestamp"
             }
         }
     ]