Update Filebeat Traefik module to handle mixed Common and Combined Log Format

CHANGELOG.asciidoc

@@ -79,6 +79,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d
 
 - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210]
 - Added `syslog_host` variable to HAProxy module to allow syslog listener to bind to configured host. {pull}9366[9366]
+- Added support on Traefik for Common Log Format and Combined Log Format mixed which is the default Traefik format {issue}8015[8015] {issue}6111[6111] {pull}8768[8768].
 
 *Heartbeat*
 

filebeat/docs/fields.asciidoc

@@ -6329,6 +6329,16 @@ type: keyword
 Client IP address.
 
 
+--
+
+*`traefik.access.duration`*::
++
+--
+type: long
+
+Duration of the access entry.
+
+
 --
 
 *`traefik.access.user_name`*::
@@ -6339,6 +6349,16 @@ type: keyword
 The user name used when basic authentication is used.
 
 
+--
+
+*`traefik.access.user_identifier`*::
++
+--
+type: keyword
+
+Is the RFC 1413 identity of the client
+
+
 --
 
 *`traefik.access.method`*::
@@ -6430,6 +6450,16 @@ type: keyword
 The name of the physical device.
 
 
+--
+
+*`traefik.access.user_agent.build`*::
++
+--
+type: keyword
+
+The build of the user agent.
+
+
 --
 
 *`traefik.access.user_agent.major`*::

filebeat/module/traefik/access/_meta/fields.yml

@@ -7,10 +7,18 @@
       type: keyword
       description: >
         Client IP address.
+    - name: duration
+      type: long
+      description: >
+        Duration of the access entry.
     - name: user_name
       type: keyword
       description: >
         The user name used when basic authentication is used.
+    - name: user_identifier
+      type: keyword
+      description: >
+        Is the RFC 1413 identity of the client
     - name: method
       type: keyword
       example: GET
@@ -52,6 +60,10 @@
           type: keyword
           description: >
             The name of the physical device.
+        - name: build
+          type: keyword
+          description: >
+            The build of the user agent.
         - name: major
           type: long
           description: >

filebeat/module/traefik/access/config/traefik-access.yml

@@ -7,7 +7,7 @@ exclude_files: [".gz$"]
 
 processors:
 - dissect:
-    tokenizer: '%{traefik.access.remote_ip} - %{traefik.access.user_name} [%{traefik.access.time}]
+    tokenizer: '%{traefik.access.remote_ip} %{traefik.access.user_identifier} %{traefik.access.user_name} [%{traefik.access.time}]
     "%{traefik.access.method} %{traefik.access.url} HTTP/%{traefik.access.http_version}"
     %{traefik.access.response_code} %{traefik.access.message}'
 

filebeat/module/traefik/access/ingest/pipeline.json

@@ -5,19 +5,21 @@
             "grok": {
                 "field": "traefik.access.message",
                 "patterns": [
-                    "(?:%{NUMBER:traefik.access.body_sent.bytes}|-)( \"%{DATA:traefik.access.referrer}\")?( \"%{DATA:traefik.access.agent}\")?(?:%{NUMBER:traefik.access.request_count}|-)?( \"%{DATA:traefik.access.frontend_name}\")?( \"%{DATA:traefik.access.backend_url}\")?"
+                    "(?:%{NUMBER:traefik.access.body_sent.bytes:int}|-)( (?:\"%{DATA:traefik.access.referrer}\"|-)?( (?:\"%{DATA:traefik.access.agent}\"|-)?)?( (?:%{NUMBER:traefik.access.request_count:int}|-)?)?( (?:\"%{DATA:traefik.access.frontend_name}\"|-)?)?( \"%{DATA:traefik.access.backend_url}\")?( %{NUMBER:traefik.access.duration:int}ms)?)?"
                 ],
                 "ignore_missing": true
             }
         },
         {
             "remove": {
-                "field": "message"
+                "field": "message",
+                "ignore_missing": true
             }
         },
         {
             "remove": {
-                "field": "traefik.access.message"
+                "field": "traefik.access.message",
+                "ignore_missing": true
             }
         },
         {
@@ -50,7 +52,8 @@
         {
             "rename": {
                 "field": "traefik.access.agent",
-                "target_field": "traefik.access.user_agent.original"
+                "target_field": "traefik.access.user_agent.original",
+                "ignore_failure": true
             }
         },
         {

filebeat/module/traefik/access/test/test.log

@@ -1,2 +1,7 @@
 192.168.33.1 - - [02/Oct/2017:20:22:07 +0000] "GET /ui/favicons/favicon-16x16.png HTTP/1.1" 304 0 "http://example.com/login" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" 262 "Host-host-1" "http://172.19.0.3:5601" 2ms
-85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] "GET /ui/favicons/favicon.ico HTTP/1.1" 304 0 "http://example.com/login" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" 271 "Host-host1 "http://172.19.0.3:5601" 3ms
+85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] "GET /ui/favicons/favicon.ico HTTP/1.1" 304 0 "http://example.com/login" "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" 271 "Host-host1" "http://172.19.0.3:5601" 3ms
+70.29.80.15 - - [28/Feb/2018:17:30:33 +0000] "GET /en/ HTTP/2.0" 200 2814 - "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1" 13 "Host-host1-com-0" "http://172.19.0.6:14008" 247ms
+::1 - - [29/Nov/2018:15:03:51 +0000] "GET / HTTP/1.1" 404 19 "-" "curl/7.62.0" 10 "backend not found" "/" 0ms
+94.254.131.115 - - [19/Jan/2018:10:01:02 +0000] "GET /assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo HTTP/1.1" 200 85 - "Android" 623112 "Host-api-wearerealitygames-com-2" "http://172.25.0.9:4140" 13ms
+89.64.35.193 - - [19/Jan/2018:10:01:02 +0000] "GET /marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM HTTP/1.1" 200 150 - "Android" 623114 "Host-api-wearerealitygames-com-2" "http://172.25.0.6:4140" 8ms
+127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326

filebeat/module/traefik/access/test/test.log-expected.json

@@ -5,11 +5,15 @@
         "event.module": "traefik", 
         "input.type": "log", 
         "log.offset": 0, 
-        "traefik.access.body_sent.bytes": "0", 
+        "traefik.access.backend_url": "http://172.19.0.3:5601", 
+        "traefik.access.body_sent.bytes": 0, 
+        "traefik.access.duration": 2, 
+        "traefik.access.frontend_name": "Host-host-1", 
         "traefik.access.http_version": "1.1", 
         "traefik.access.method": "GET", 
         "traefik.access.referrer": "http://example.com/login", 
         "traefik.access.remote_ip": "192.168.33.1", 
+        "traefik.access.request_count": 262, 
         "traefik.access.response_code": "304", 
         "traefik.access.url": "/ui/favicons/favicon-16x16.png", 
         "traefik.access.user_agent.device": "Other", 
@@ -20,6 +24,7 @@
         "traefik.access.user_agent.os": "Linux", 
         "traefik.access.user_agent.os_name": "Linux", 
         "traefik.access.user_agent.patch": "3163", 
+        "traefik.access.user_identifier": "-", 
         "traefik.access.user_name": "-"
     }, 
     {
@@ -28,7 +33,10 @@
         "event.module": "traefik", 
         "input.type": "log", 
         "log.offset": 280, 
-        "traefik.access.body_sent.bytes": "0", 
+        "traefik.access.backend_url": "http://172.19.0.3:5601", 
+        "traefik.access.body_sent.bytes": 0, 
+        "traefik.access.duration": 3, 
+        "traefik.access.frontend_name": "Host-host1", 
         "traefik.access.geoip.city_name": "Berlin", 
         "traefik.access.geoip.continent_name": "Europe", 
         "traefik.access.geoip.country_iso_code": "DE", 
@@ -40,6 +48,7 @@
         "traefik.access.method": "GET", 
         "traefik.access.referrer": "http://example.com/login", 
         "traefik.access.remote_ip": "85.181.35.98", 
+        "traefik.access.request_count": 271, 
         "traefik.access.response_code": "304", 
         "traefik.access.url": "/ui/favicons/favicon.ico", 
         "traefik.access.user_agent.device": "Other", 
@@ -50,6 +59,147 @@
         "traefik.access.user_agent.os": "Linux", 
         "traefik.access.user_agent.os_name": "Linux", 
         "traefik.access.user_agent.patch": "3163", 
+        "traefik.access.user_identifier": "-", 
         "traefik.access.user_name": "-"
+    }, 
+    {
+        "@timestamp": "2018-02-28T17:30:33.000Z", 
+        "event.dataset": "access", 
+        "event.module": "traefik", 
+        "input.type": "log", 
+        "log.offset": 553, 
+        "traefik.access.backend_url": "http://172.19.0.6:14008", 
+        "traefik.access.body_sent.bytes": 2814, 
+        "traefik.access.duration": 247, 
+        "traefik.access.frontend_name": "Host-host1-com-0", 
+        "traefik.access.geoip.city_name": "Ottawa", 
+        "traefik.access.geoip.continent_name": "North America", 
+        "traefik.access.geoip.country_iso_code": "CA", 
+        "traefik.access.geoip.location.lat": 45.2691, 
+        "traefik.access.geoip.location.lon": -75.7518, 
+        "traefik.access.geoip.region_iso_code": "CA-ON", 
+        "traefik.access.geoip.region_name": "Ontario", 
+        "traefik.access.http_version": "2.0", 
+        "traefik.access.method": "GET", 
+        "traefik.access.remote_ip": "70.29.80.15", 
+        "traefik.access.request_count": 13, 
+        "traefik.access.response_code": "200", 
+        "traefik.access.url": "/en/", 
+        "traefik.access.user_agent.device": "iPhone", 
+        "traefik.access.user_agent.major": "11", 
+        "traefik.access.user_agent.minor": "0", 
+        "traefik.access.user_agent.name": "Mobile Safari", 
+        "traefik.access.user_agent.original": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_2_5 like Mac OS X) AppleWebKit/604.5.6 (KHTML, like Gecko) Version/11.0 Mobile/15D60 Safari/604.1", 
+        "traefik.access.user_agent.os": "iOS 11.2.5", 
+        "traefik.access.user_agent.os_major": "11", 
+        "traefik.access.user_agent.os_minor": "2", 
+        "traefik.access.user_agent.os_name": "iOS", 
+        "traefik.access.user_identifier": "-", 
+        "traefik.access.user_name": "-"
+    }, 
+    {
+        "@timestamp": "2018-11-29T15:03:51.000Z", 
+        "event.dataset": "access", 
+        "event.module": "traefik", 
+        "input.type": "log", 
+        "log.offset": 821, 
+        "traefik.access.backend_url": "/", 
+        "traefik.access.body_sent.bytes": 19, 
+        "traefik.access.duration": 0, 
+        "traefik.access.frontend_name": "backend not found", 
+        "traefik.access.http_version": "1.1", 
+        "traefik.access.method": "GET", 
+        "traefik.access.referrer": "-", 
+        "traefik.access.remote_ip": "::1", 
+        "traefik.access.request_count": 10, 
+        "traefik.access.response_code": "404", 
+        "traefik.access.url": "/", 
+        "traefik.access.user_agent.device": "Other", 
+        "traefik.access.user_agent.major": "7", 
+        "traefik.access.user_agent.minor": "62", 
+        "traefik.access.user_agent.name": "curl", 
+        "traefik.access.user_agent.original": "curl/7.62.0", 
+        "traefik.access.user_agent.os": "Other", 
+        "traefik.access.user_agent.os_name": "Other", 
+        "traefik.access.user_agent.patch": "0", 
+        "traefik.access.user_identifier": "-", 
+        "traefik.access.user_name": "-"
+    }, 
+    {
+        "@timestamp": "2018-01-19T10:01:02.000Z", 
+        "event.dataset": "access", 
+        "event.module": "traefik", 
+        "input.type": "log", 
+        "log.offset": 931, 
+        "traefik.access.backend_url": "http://172.25.0.9:4140", 
+        "traefik.access.body_sent.bytes": 85, 
+        "traefik.access.duration": 13, 
+        "traefik.access.frontend_name": "Host-api-wearerealitygames-com-2", 
+        "traefik.access.geoip.city_name": "Warsaw", 
+        "traefik.access.geoip.continent_name": "Europe", 
+        "traefik.access.geoip.country_iso_code": "PL", 
+        "traefik.access.geoip.location.lat": 52.25, 
+        "traefik.access.geoip.location.lon": 21.0, 
+        "traefik.access.geoip.region_iso_code": "PL-MZ", 
+        "traefik.access.geoip.region_name": "Mazovia", 
+        "traefik.access.http_version": "1.1", 
+        "traefik.access.method": "GET", 
+        "traefik.access.remote_ip": "94.254.131.115", 
+        "traefik.access.request_count": 623112, 
+        "traefik.access.response_code": "200", 
+        "traefik.access.url": "/assets/52f8f2e711d235d76044799e/owners?oauth_token=ya29.GltABOXd_gtG-XVvYX2YhxXJiXVvbHRMXn9fbzc_mDfl2rDhqK0CrAlwuwwRWnNnEaMDwkmyI7-QGbRSB0Hzje2cc__FjTQ1iuiYTSIBaIPfxSWip5jx6zqvsVVo", 
+        "traefik.access.user_agent.device": "Generic Smartphone", 
+        "traefik.access.user_agent.name": "Other", 
+        "traefik.access.user_agent.original": "Android", 
+        "traefik.access.user_agent.os": "Android", 
+        "traefik.access.user_agent.os_name": "Android", 
+        "traefik.access.user_identifier": "-", 
+        "traefik.access.user_name": "-"
+    }, 
+    {
+        "@timestamp": "2018-01-19T10:01:02.000Z", 
+        "event.dataset": "access", 
+        "event.module": "traefik", 
+        "input.type": "log", 
+        "log.offset": 1267, 
+        "traefik.access.backend_url": "http://172.25.0.6:4140", 
+        "traefik.access.body_sent.bytes": 150, 
+        "traefik.access.duration": 8, 
+        "traefik.access.frontend_name": "Host-api-wearerealitygames-com-2", 
+        "traefik.access.geoip.city_name": "Katowice", 
+        "traefik.access.geoip.continent_name": "Europe", 
+        "traefik.access.geoip.country_iso_code": "PL", 
+        "traefik.access.geoip.location.lat": 50.2194, 
+        "traefik.access.geoip.location.lon": 18.9737, 
+        "traefik.access.geoip.region_iso_code": "PL-SL", 
+        "traefik.access.geoip.region_name": "Silesia", 
+        "traefik.access.http_version": "1.1", 
+        "traefik.access.method": "GET", 
+        "traefik.access.remote_ip": "89.64.35.193", 
+        "traefik.access.request_count": 623114, 
+        "traefik.access.response_code": "200", 
+        "traefik.access.url": "/marketplace/tax?oauth_token=ya29.Gl0fBWnrJ7DcEU-tN-O3Vxn2XZVaz2I-hFTjP1JQzhYFVT-SKtlmo9hSzrx3n82LUwUxJ1s5lmU8U3Mc9gA_aCxBk49ShYEwvmYOWxJJyldDIJ7hY4us4LoiSY1OqAM", 
+        "traefik.access.user_agent.device": "Generic Smartphone", 
+        "traefik.access.user_agent.name": "Other", 
+        "traefik.access.user_agent.original": "Android", 
+        "traefik.access.user_agent.os": "Android", 
+        "traefik.access.user_agent.os_name": "Android", 
+        "traefik.access.user_identifier": "-", 
+        "traefik.access.user_name": "-"
+    }, 
+    {
+        "@timestamp": "2000-10-10T20:55:36.000Z", 
+        "event.dataset": "access", 
+        "event.module": "traefik", 
+        "input.type": "log", 
+        "log.offset": 1581, 
+        "traefik.access.body_sent.bytes": 2326, 
+        "traefik.access.http_version": "1.0", 
+        "traefik.access.method": "GET", 
+        "traefik.access.remote_ip": "127.0.0.1", 
+        "traefik.access.response_code": "200", 
+        "traefik.access.url": "/apache_pb.gif", 
+        "traefik.access.user_identifier": "-", 
+        "traefik.access.user_name": "frank"
     }
 ]