diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index 4accdae0f35..8937d638025 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -450,6 +450,16 @@ ## HAProxy module +- from: haproxy.total_waiting_time_ms + to: event.duration + alias: false + scale: 1000000 + +- from: haproxy.http.request.time_active_ms + to: event.duration + alias: false + scale: 1000000 + - from: haproxy.client.ip to: source.address alias: true diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 066158e04a9..6ac00d260eb 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4899,15 +4899,6 @@ Complete HTTP request line, including the method, request and HTTP version strin -- -*`haproxy.http.request.time_active_ms`*:: -+ --- -type: long - -Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent. - --- - *`haproxy.http.request.time_wait_without_data_ms`*:: + -- @@ -4932,15 +4923,6 @@ Total time in milliseconds spent waiting for a full HTTP request from the client TCP log format -*`haproxy.tcp.processing_time_ms`*:: -+ --- -type: long - -Total time in milliseconds elapsed between the accept and the last close - --- - *`haproxy.tcp.connection_waiting_time_ms`*:: + -- diff --git a/filebeat/module/haproxy/fields.go b/filebeat/module/haproxy/fields.go index 1797a39e9d9..4b36485924f 100644 --- a/filebeat/module/haproxy/fields.go +++ b/filebeat/module/haproxy/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzMWdtu2zwSvs9TDHKzLZD6AQJsgUW2uy3w/20ucm/Q5FgalCJVcpTUb/+DB8k62lbiovWdRc7MxzkP+QG+4+EeSlE7+/NwA8DEGu/hNn+5vQFQ6KWjmsmae/h4AwDtfvjbqkbjDcCeUCt/Hxc/gBEV9pmGHx9qvIfC2abOX2b4Hhnlv0due2cNo1Hb8LdbHXH5KioEuwcusSOAd9aBJs9o0L2Hl5JkCQ4l0jMqEEZB7axE71FFOmmNQRn4baYodkJ+XwMi75/F8CI8eNQoOUi2UAkjChxhCAvhi0f3jG4GUVq4GJAWnjNNYJ2QjEQmYIZnpLFlobcvgphMsWWqcFv5JblPYTOETUAGKtKaPEprlAdfo2HIfMJqgPAsHNnGw48GG+xzTb6jrSmmiI64I6yrYdpbN2MK9Cx2mnzZ2mVPRuis0AsR7w6MfutQqNMgTVPt0AXDRQpgJ4yviLO7RHCaIuYSkwa1LYA8YNq1uRBQVFlU+q+x5CocbYRlvf8OO94BGakbFcgdsqPLj5Cj8QJlHq3r8EeDnn2bFtBhLyXtcG9dyArkwRpsNZxDOAq6FFyr2l+PLkv6l4dC253QK3HSigybsmowlVDKoffjDH82p6Nz1m0r9F4UiyI/hU2QN4U4K8LpD/D5P4+xEpIBKXxEFflNz8r4k2ccxjZO4vzmJQuV2IlN5J0u7FxUoavIiJghPQtePOKDNYpSgEQH874tBmSOKab9jkaFDDMRV1m17jiBALgUPKzawaFqdCIG8bunh0ewDj4/PT2+P1UDFvP+gzUsyPguNUnbGPZBcz1qEJLpuXPj7OdTW/bbGIBh+zMGl1gOls6EnLRGNs6FLNbHZgegphYJlkqOuRlJmws2mOmtrgeyM+MVUeakcgWQpTBKp/jtN2lXxDppCF4L1TNp3bqlHeT+K8LNVe4U3q99pG0lzXSAP2t0hEa2SiV/RBVhukOIY7Yt9bivPY+3C/bY8yzF+Rezt66K2Q7Ezjbc75OUjSBKbKvaKwK7hUH1zRSx0CSGWqwFl22W3+QKNdhQUeFEws5uUJZbSbV1vF7WhGosaKLYnFjGlXdeWpLVJsgRyXlRpFZKGFKcFaDQcy56S57y3+MWoKPXvMElVhqqh/G8tabSVjlgX9aI8KwuC7QDknm1zE7z6dfV3v+j/fLYVzYUgkt0IWuI0E7mBJfCdUN1UvYwO3wz+gC1wzChAqXGJzH+FIZbkh6FkyXUuinIhD5CPAvSYqcxTPsDXo0f5spTdVxaw2TQ8DhATml/FJQF2s1JPstm7wNpDLvDlrzdymG3tRrKSU6XgNFWjsNmJYgFDpcId1iErvZt9lhmcpExiA9vdYgFFis08HZXOM1oCUp31ce8mAseNYbBSCjV/746zc5kQIe+tsYPwQ5k/y8yAYda9G5NwhQxTzwP5UxeEDU3DtVWWvudTjb6H0eLAN/iitBwG5j9+1noBm8BQ1QCGUUyzT/deJQbmVKo1GQlme280p5pM6e1GcQlCoXuZNs3hfwXeQ5tYCbuuI1BgGqw1XhK18dB9TYTHTcnbrcQJ9QqpvbhJDHfIg4Gy/lOnAU3/jXREZx606l0mc9ScMx67LjtXOOwU9o/3l/zqFKKcCZunEEFovXaF+ISiH17tN/sthHDhV6b9v4yp3XiZZuFbDWZk0Z6sFWtkYdeAoGqf48ZUFXIpVV33R5hVCJ6RhdHNc+OTPFa0PH6Nk2qwwv4CeQnqnCgdIeVoOgb3f1Lfj26y9d55CMBL1/7ohZ1upLkF8yT8Z6c53h93tqwlfgSPbL3CNQ9jrS7u9R0fA6Z00scUs/rJb5NBIe3DW+VYHFORWvvt49vOh5NCLJ9o/Ww1N2BsZzax0AZULz9TFc8xhBzstPe2apf+d4NjrCz6vAexJ7Rje3dN/CaU3YnlIsdzdPDY3xpSWPMWwbGNNzOP6XNwlyj4LmAEFJizUOHl9r6uQFz9LL2R6D8JwAA///iRBtm" + return "eJzMWc2O2zYQvu9TDPbSBEj8AAs0QLFNmwBtsoe9G2NyLBGhSIUc7cZvX/BHsn5ta9dB6pslzszH+fk4Q72Hb3S4gxJrZ38cbgBYsaY7uM1Pbm8AJHnhVM3Kmjv4cAMA7Xr418pG0w3AXpGW/i6+fA8GK+orDT8+1HQHhbNNnZ/M6D0qyn+P2vbOGiYjt+Fv93ak5QtWBHYPXFInAG+sA608kyH3Fp5LJUpwJEg9kQQ0EmpnBXlPMsoJawyJoG8zRbFD8W0NiLx+FsMzevCkSXCwbKFCgwWNMIQX4Ykn90RuBlF6cTEgjZ6zTFCdkIxMJmCGZ6yxZdTbZ1SsTLFlVdG28kt2H8NiCItAGaiU1sqTsEZ68DUZhqwnvA0QntAp23j43lBDfa0pd7Q1xRTREXeEdTVMe+tmQkGecaeVL9u47JVBnR16IeLdgclvHaE8DdI01Y5cCFyUAHZofKU4p0sEp1XEXFLyoLYFKA+UVm0uBBRdFp3+cyK5CkdbYdnvvyKO70AZoRsZxB2xU5dvIVfjBc48RtfR94Y8+5YWyFGPkna0ty6wgvJgDbUeziUcDV0KrnXtz0eXLf3modB2h3olTrWCYROrhlChlI68HzP8WU4n56zbVuQ9FosmP4ZFkBeFOivC7g/w6Y+HeBIqAwJ9RBX1TffK9INnEsY2TtD84qUIldSZTeKdL+xcVZGrlMHIkJ6RF7d4b41UqUBignnfHgbKHCmmfU5GBoaZmKusXLedIABcIg9P7ZBQNTmMRfzm8f4BrINPj48Pb0+dAYu8f28NozK+oyZhG8M+eK4nDShYPXVpnPN8Gst+GwMwbH/G4JLKwaszJSesEY1zgcX62OwA1DQiIVIpMTcja3PFBjO91fVAdmG8IspMKlcAWaKROtVvv0m7ItZJQ/BSqJ6V1m1a2gH3XxFuPuVO4f3SR9qepFkO6EdNTpERrVOVP6KKMN0h1DHbVnrc157H2xV77HmW6vyz2VtXRbYD3NmG+32StBFESe2p9oLCbmGo+maKGLXCoRdr5LJl+U0+oQYLKlU4TNjZDY7l1lJtHa+3NZEaG5o4NhPL+OSdt5ZstQQ5EjlvSsmVFoYSZw1I8pwPvaVM+fO4BNQxa16REisD1cN4PlpTa6sSsG9rJHjWlwXZgci8W2an+fTrzt6/yX5+6DsbCuSSXGANDO1kJrhUrhtVJ2cP2eGr0QeoHYUJFVRqfJLij2G4VcITOlFCrZtCmdBH4BMqjTtNYdof6Gr8kCtPnePCGlaGDI8L5JT3R0VZkN2c1LMc9j6QxrA7bJW3WzHstlZDOanpEjDainHZrASxoOES446K0NW+Lh7LSi4KhuLDaxNiQcUKD7w+FU4rWoLSXfUxL3LBg6YwGKGU/eeraXaGAR352ho/BDuw/VdUAo409m5NwhQxLzwP5QwvYM2NI7kV1n5TJxv9D6OXAF/jG9RwG5T9/oS6oVugUJWgjFQizT/deJQbmRJlarKSzXZeafe0mfPaDOKSUJI72fZNIf+jPIc2MAt32sYgQDbUejzR9XFQvc1Cx8VJ2y3ECbWK1D6cJOZbxMFgOd+JM3LjX1IdIak3nUuX9SwVx2zGjtvONQk7lf3f52seVUoMe+LGGZKAbdY+Ky5BsW+39ovTNmK4MGvT2p+WtA6ft9nIVitzMkj3tqo18TBLIEj17zEDqoq4tPJdtwaNTEJP5OKo5tkpU7wUdLy+jXfwIbC24a1ExuFd/AT96nvc47cLH2Z2hH2j9ZDS34GxnNqkIBlQnBgwL9zTFbcxxJxisXe26jP8m8EWdlYe3gLumVy+rXae42eBOPS3d51rdtntUCye3I/3D/GLQmrXXzEYjb7RzH86moW7xtGksU530vxM+WoEhaA6ZXr39Uto6+nmvwAAAP//o2OPYg==" } diff --git a/filebeat/module/haproxy/log/_meta/fields.yml b/filebeat/module/haproxy/log/_meta/fields.yml index 6a7c8e8f477..9176a8ae728 100644 --- a/filebeat/module/haproxy/log/_meta/fields.yml +++ b/filebeat/module/haproxy/log/_meta/fields.yml @@ -38,10 +38,6 @@ description: Complete HTTP request line, including the method, request and HTTP version string. type: text - - name: time_active_ms - description: Time the request remained active in haproxy, which is the total time in milliseconds elapsed between the first byte of the request was received and the last byte of response was sent. - type: long - - name: time_wait_without_data_ms description: Total time in milliseconds spent waiting for the server to send a full HTTP response, not counting data. type: long @@ -54,9 +50,6 @@ description: TCP log format type: group fields: - - name: processing_time_ms - type: long - description: Total time in milliseconds elapsed between the accept and the last close - name: connection_waiting_time_ms type: long description: Total time in milliseconds elapsed between the accept and the last close diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index 08f4315c88e..38ed994296d 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -7,11 +7,11 @@ "patterns": [ "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYDATA} %{IPORHOST:source.address}:%{POSINT:source.port:long} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:long} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:haproxy.http.request.time_active_ms:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:long}/%{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:http.response.status_code:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long} (\\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} |\\{%{DATA}\\} )?\"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", - "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:haproxy.tcp.processing_time_ms:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" + "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:long}\\]: )?%{IP:source.address}:%{NUMBER:source.port:long} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:long}/%{NUMBER:haproxy.connection_wait_time_ms:long}/%{NUMBER:temp.duration:long} %{NUMBER:haproxy.bytes_read:long} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:long}/%{NUMBER:haproxy.connections.frontend:long}/%{NUMBER:haproxy.connections.backend:long}/%{NUMBER:haproxy.connections.server:long}/%{NUMBER:haproxy.connections.retries:long} %{NUMBER:haproxy.server_queue:long}/%{NUMBER:haproxy.backend_queue:long}" ], "ignore_missing": false, "pattern_definitions": { @@ -73,7 +73,15 @@ { "script": { "lang": "painless", - "source": "if (ctx.haproxy.http?.request?.time_active_ms != null) { ctx.event.duration = ctx.haproxy.http.request.time_active_ms * 1000000 } else { if (ctx.haproxy.tcp?.processing_time_ms != null) { ctx.event.duration = ctx.haproxy.tcp.processing_time_ms * 1000000 } }" + "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)", + "params": { "scale": 1000000 }, + "if": "ctx.temp?.duration != null" + } + }, + { + "remove": { + "field": "temp.duration", + "ignore_missing": true } }, diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index b269eb0e09f..6ebe02ae203 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -20,7 +20,6 @@ "docs.example.internal" ], "haproxy.http.request.raw_request_line": "GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1", - "haproxy.http.request.time_active_ms": 2, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": 0, "haproxy.http.response.captured_cookie": "-", diff --git a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json index 5f80d850cfc..970b30f90bf 100644 --- a/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json +++ b/filebeat/module/haproxy/log/test/httplog-no-headers.log-expected.json @@ -17,7 +17,6 @@ "haproxy.frontend_name": "http-webservices", "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.raw_request_line": "GET / HTTP/1.1", - "haproxy.http.request.time_active_ms": 0, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": -1, "haproxy.http.response.captured_cookie": "-", @@ -54,7 +53,6 @@ "haproxy.frontend_name": "http-webservices", "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", - "haproxy.http.request.time_active_ms": 0, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": -1, "haproxy.http.response.captured_cookie": "-", @@ -94,7 +92,6 @@ "localhost:8888" ], "haproxy.http.request.raw_request_line": "GET /foo HTTP/1.1", - "haproxy.http.request.time_active_ms": 0, "haproxy.http.request.time_wait_ms": 0, "haproxy.http.request.time_wait_without_data_ms": -1, "haproxy.http.response.captured_cookie": "-", diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index d5ffb2e3f13..439e4dc2168 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -18,7 +18,6 @@ "haproxy.server_name": "", "haproxy.server_queue": 0, "haproxy.source": "127.0.0.1", - "haproxy.tcp.processing_time_ms": 1, "haproxy.termination_state": "SC", "haproxy.total_waiting_time_ms": -1, "input.type": "log",