Complete the migration to event.duration for haproxy.log.

elastic · Jan 23, 2019 · c3e592a · c3e592a
1 parent 3d4c48a
commit c3e592a
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 34 deletions.
diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -450,6 +450,16 @@
 
 ## HAProxy module
 
+- from: haproxy.total_waiting_time_ms
+  to: event.duration
+  alias: false
+  scale: 1000000
+
+- from: haproxy.http.request.time_active_ms
+  to: event.duration
+  alias: false
+  scale: 1000000
+
 - from: haproxy.client.ip
   to: source.address
   alias: true

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -4899,15 +4899,6 @@ Complete HTTP request line, including the method, request and HTTP version strin
 
 --
 
-*`haproxy.http.request.time_active_ms`*::
-+
---
-type: long
-
-Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent.
-
---
-
 *`haproxy.http.request.time_wait_without_data_ms`*::
 +
 --
@@ -4932,15 +4923,6 @@ Total time in milliseconds spent waiting for a full HTTP request from the client
 TCP log format
 
 
-*`haproxy.tcp.processing_time_ms`*::
-+
---
-type: long
-
-Total time in milliseconds elapsed between the accept and the last close
-
---
-
 *`haproxy.tcp.connection_waiting_time_ms`*::
 +
 --

diff --git a/filebeat/module/haproxy/fields.go b/filebeat/module/haproxy/fields.go
diff --git a/filebeat/module/haproxy/log/_meta/fields.yml b/filebeat/module/haproxy/log/_meta/fields.yml
@@ -38,10 +38,6 @@
       description: Complete HTTP request line, including the method, request and HTTP version string.
       type: text
 
-    - name: time_active_ms
-      description: Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent.
-      type: long
-
     - name: time_wait_without_data_ms
       description: Total time in milliseconds spent waiting for the server to send a full HTTP response, not counting data.
       type: long
@@ -54,9 +50,6 @@
   description: TCP log format
   type: group
   fields:
-  - name: processing_time_ms
-    type: long
-    description: Total time in milliseconds elapsed between the accept and the last close
   - name: connection_waiting_time_ms
     type: long
     description: Total time in milliseconds elapsed between the accept and the last close
diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json
@@ -7,11 +7,11 @@
                 "patterns": [
                     "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)",
 
-                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
+                    "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"",
 
                     "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}",
 
-                    "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}"
+                    "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}"
                 ],
                 "ignore_missing": false,
                 "pattern_definitions": {
@@ -73,7 +73,15 @@
         {
             "script": {
                 "lang": "painless",
-                "source": "if (ctx.haproxy.http?.request?.time_active_ms != null) { ctx.event.duration = ctx.haproxy.http.request.time_active_ms * 1000000 } else { if (ctx.haproxy.tcp?.processing_time_ms != null) { ctx.event.duration = ctx.haproxy.tcp.processing_time_ms * 1000000 } }"
+                "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)",
+                "params": { "scale": 1000000 },
+                "if": "ctx.temp?.duration != null"
+            }
+        },
+        {
+            "remove": {
+                "field": "temp.duration",
+                "ignore_missing": true
             }
         },
 

diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json
@@ -20,7 +20,6 @@
             "docs.example.internal"
         ],
         "haproxy.http.request.raw_request_line": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1",
-        "haproxy.http.request.time_active_ms": 2,
         "haproxy.http.request.time_wait_ms": 0,
         "haproxy.http.request.time_wait_without_data_ms": 0,
         "haproxy.http.response.captured_cookie": "-",

diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json
@@ -17,7 +17,6 @@
         "haproxy.frontend_name": "http-webservices",
         "haproxy.http.request.captured_cookie": "-",
         "haproxy.http.request.raw_request_line": "GET / HTTP/1.1",
-        "haproxy.http.request.time_active_ms": 0,
         "haproxy.http.request.time_wait_ms": 0,
         "haproxy.http.request.time_wait_without_data_ms": -1,
         "haproxy.http.response.captured_cookie": "-",
@@ -54,7 +53,6 @@
         "haproxy.frontend_name": "http-webservices",
         "haproxy.http.request.captured_cookie": "-",
         "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1",
-        "haproxy.http.request.time_active_ms": 0,
         "haproxy.http.request.time_wait_ms": 0,
         "haproxy.http.request.time_wait_without_data_ms": -1,
         "haproxy.http.response.captured_cookie": "-",
@@ -94,7 +92,6 @@
             "localhost:8888"
         ],
         "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1",
-        "haproxy.http.request.time_active_ms": 0,
         "haproxy.http.request.time_wait_ms": 0,
         "haproxy.http.request.time_wait_without_data_ms": -1,
         "haproxy.http.response.captured_cookie": "-",

diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json
@@ -18,7 +18,6 @@
         "haproxy.server_name": "<NOSRV>",
         "haproxy.server_queue": 0,
         "haproxy.source": "127.0.0.1",
-        "haproxy.tcp.processing_time_ms": 1,
         "haproxy.termination_state": "SC",
         "haproxy.total_waiting_time_ms": -1,
         "input.type": "log",