Add jvm9 logging format to GC fileset of ES module (#7390)

elastic · Jun 21, 2018 · 61c7203 · 61c7203
1 parent 0576fe6
commit 61c7203
Show file tree

Hide file tree

Showing 9 changed files with 5,361 additions and 13 deletions.
diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -811,7 +811,7 @@ elasticsearch Module
 --
 type: float
 
-Garbage collection threads total stop time seconds.
+The time from JVM start up in seconds, as a floating point number.
 
 
 --
@@ -834,6 +834,26 @@ type: float
 Time took to stop threads seconds.
 
 
+--
+
+*`elasticsearch.gc.pid`*::
++
+--
+type: long
+
+The PID of JVM process.
+
+
+--
+
+*`elasticsearch.gc.tags`*::
++
+--
+type: keyword
+
+GC logging tags.
+
+
 --
 
 [float]

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/module/elasticsearch/gc/_meta/fields.yml b/filebeat/module/elasticsearch/gc/_meta/fields.yml
@@ -5,7 +5,7 @@
     - name: relative_process_timestamp_secs
       type: float
       description: >
-        Garbage collection threads total stop time seconds.
+        The time from JVM start up in seconds, as a floating point number.
     - name: threads_total_stop_time_secs
       type: float
       description: >
@@ -14,3 +14,11 @@
       type: float
       description: >
         Time took to stop threads seconds.
+    - name: pid
+      type: long
+      description: >
+        The PID of JVM process.
+    - name: tags
+      type: keyword
+      description: >
+        GC logging tags.
diff --git a/filebeat/module/elasticsearch/gc/config/gc.yml b/filebeat/module/elasticsearch/gc/config/gc.yml
@@ -4,11 +4,12 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+exclude_lines: ["^(OpenJDK|Java HotSpot).* Server VM ", "^CommandLine flags: ", "^Memory: "] # exclude JVM8 banner
 multiline:
-  pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
+  pattern: '^\[?[0-9]{4}-[0-9]{2}-[0-9]{2}'
   negate: true
   match: after
 
 fields:
   service.name: "elasticsearch"
-fields_under_root: true
+fields_under_root: true
diff --git a/filebeat/module/elasticsearch/gc/ingest/pipeline.json b/filebeat/module/elasticsearch/gc/ingest/pipeline.json
@@ -5,13 +5,13 @@
       "grok": {
         "field": "message",
         "patterns": [
-          "%{GCTIMESTAMP}: %{GCPROCRUNTIME}: Total time for which application threads were stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_secs} seconds, Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_secs} seconds",
-          "%{GCTIMESTAMP}: %{GREEDYMULTILINE:message}"
+          "(?:%{JVM8HEADER}|%{JVM9HEADER}) Total time for which application threads were stopped: %{BASE10NUM:elasticsearch.gc.threads_total_stop_time_secs} seconds, Stopping threads took: %{BASE10NUM:elasticsearch.gc.stopping_threads_time_secs} seconds",
+          "(?:%{JVM8HEADER}|%{JVM9HEADER}) %{GREEDYMULTILINE:message}"
         ],
         "pattern_definitions": {
           "GREEDYMULTILINE": "(.|\n)*",
-          "GCTIMESTAMP": "%{TIMESTAMP_ISO8601:timestamp}",
-          "GCPROCRUNTIME": "%{BASE10NUM:elasticsearch.gc.relative_process_timestamp_secs}"
+          "JVM8HEADER": "%{TIMESTAMP_ISO8601:timestamp}: %{BASE10NUM:elasticsearch.gc.relative_process_timestamp_secs}:",
+          "JVM9HEADER": "\\[%{TIMESTAMP_ISO8601:timestamp}\\]\\[%{POSINT:elasticsearch.gc.pid}\\]\\[%{DATA:elasticsearch.gc.tags}%{SPACE}*\\]"
         }
       }
     },
@@ -22,9 +22,24 @@
       }
     },
     {
-      "rename": {
+      "date": {
         "field": "timestamp",
-        "target_field": "@timestamp"
+        "target_field": "@timestamp",
+        "formats": [
+          "ISO8601"
+        ]
+      }
+    },
+    {
+      "remove": {
+        "field": "timestamp"
+      }
+    },
+    {
+      "split": {
+        "field": "elasticsearch.gc.tags",
+        "separator": ",",
+        "ignore_missing": true
       }
     }
   ],
@@ -36,4 +51,4 @@
       }
     }
   ]
-}
+}
diff --git a/filebeat/module/elasticsearch/gc/manifest.yml b/filebeat/module/elasticsearch/gc/manifest.yml
@@ -4,6 +4,7 @@ var:
   - name: paths
     default:
       - /var/log/elasticsearch/gc.log.[0-9]*
+      - /var/log/elasticsearch/gc.log
     os.darwin: []
     os.windows: []
 

diff --git a/filebeat/module/elasticsearch/gc/test/gc.log b/filebeat/module/elasticsearch/gc/test/gc.log
diff --git a/filebeat/module/elasticsearch/gc/test/gc.log.1.current b/filebeat/module/elasticsearch/gc/test/gc.log.1.current
diff --git a/filebeat/module/elasticsearch/gc/test/test.log-expected.json b/filebeat/module/elasticsearch/gc/test/test.log-expected.json
@@ -18,7 +18,7 @@
       "input": {
         "type": "log"
       },
-      "@timestamp": "2018-06-11T01:53:11.382+0000",
+      "@timestamp": "2018-06-11T01:53:11.382Z",
       "elasticsearch": {
         "gc": {
           "stopping_threads_time_secs": "0.0000702",