Change url.path to use NOTSPACE grok pattern instead (#13225)

* Change url.path to use NOTSPACE grok pattern instead
elastic · Aug 23, 2019 · c66580d · c66580d
1 parent fea2db3
commit c66580d
Show file tree

Hide file tree

Showing 4 changed files with 149 additions and 4 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -132,6 +132,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix multiline pattern in Postgres which was too permissive {issue}12078[12078] {pull}13069[13069]
 - Allow path variables to be used in files loaded from modules.d. {issue}13184[13184]
 - Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303]
+- Change iis url.path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225]
 
 *Heartbeat*
 

diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json
@@ -5,15 +5,12 @@
             "grok": {
                 "field": "message",
                 "patterns": [
-                    "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATHWITHBRACKET:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
+                    "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{NOTSPACE:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
                     "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}",
                     "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} HTTP/%{NUMBER:http.version} %{NOTSPACE:user_agent.original} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:http.response.body.bytes:long} %{NUMBER:http.request.body.bytes:long} %{NUMBER:temp.duration:long}",
                     "%{TIMESTAMP_ISO8601:iis.access.time} \\[%{IPORHOST:destination.address}\\]\\(http://%{IPORHOST:destination.address}\\) %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} \\[%{IPORHOST:source.address}\\]\\(http://%{IPORHOST:source.address}\\) %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}",
                     "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.address} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:long} %{NOTSPACE:user.name} %{IPORHOST:source.address} %{NOTSPACE:user_agent.original} %{NUMBER:http.response.status_code:long} %{NUMBER:iis.access.sub_status:long} %{NUMBER:iis.access.win32_status:long} %{NUMBER:temp.duration:long}"
                 ],
-                "pattern_definitions": {
-                    "URIPATHWITHBRACKET": "(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\\-\\[\\]]*)+"
-                },
                 "ignore_missing": true
             }
         },

diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log b/filebeat/module/iis/access/test/test-iis-7.2.log
@@ -0,0 +1,5 @@
+2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..À¯..À¯..À¯..À¯..À¯../winnt/system32/cmd.exe /c+dir+c:\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 0
+2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..ÁÁ..ÁÁ..ÁÁ..ÁÁ..ÁÁ../winnt/system32/cmd.exe /c+dir+c:\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 2 46
+2018-12-31 12:02:53 10.44.0.136 GET /Director - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0
+2018-12-31 12:02:53 10.44.0.136 GET / - 443 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 401 0 0 0
+2018-12-31 12:02:53 10.44.0.136 GET /pbserver/..Áœ..Áœ..Áœ..Áœ..Áœ../winnt/system32/cmd.exe /c+dir+c:\+/OG 8080 - 10.50.6.188 Mozilla/4.0+(compatible;+MSIE+8.0;+Windows+NT+5.1;+Trident/4.0) - 404 0 64 15
diff --git a/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json b/filebeat/module/iis/access/test/test-iis-7.2.log-expected.json
@@ -0,0 +1,142 @@
+[
+    {
+        "@timestamp": "2018-12-31T12:02:53.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 8080,
+        "event.dataset": "iis.access",
+        "event.duration": 0,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 404,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 64,
+        "input.type": "log",
+        "log.offset": 0,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/pbserver/..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af..\u00c0\u00af../winnt/system32/cmd.exe",
+        "url.query": "/c+dir+c:\\+/OG",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
+    },
+    {
+        "@timestamp": "2018-12-31T12:02:53.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 8080,
+        "event.dataset": "iis.access",
+        "event.duration": 46000000,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 404,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 2,
+        "input.type": "log",
+        "log.offset": 213,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/pbserver/..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1..\u00c1\u00c1../winnt/system32/cmd.exe",
+        "url.query": "/c+dir+c:\\+/OG",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
+    },
+    {
+        "@timestamp": "2018-12-31T12:02:53.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 443,
+        "event.dataset": "iis.access",
+        "event.duration": 0,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 401,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 0,
+        "input.type": "log",
+        "log.offset": 426,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/Director",
+        "url.query": "-",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
+    },
+    {
+        "@timestamp": "2018-12-31T12:02:53.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 443,
+        "event.dataset": "iis.access",
+        "event.duration": 0,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 401,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 0,
+        "input.type": "log",
+        "log.offset": 568,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/",
+        "url.query": "-",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
+    },
+    {
+        "@timestamp": "2018-12-31T12:02:53.000Z",
+        "destination.address": "10.44.0.136",
+        "destination.ip": "10.44.0.136",
+        "destination.port": 8080,
+        "event.dataset": "iis.access",
+        "event.duration": 15000000,
+        "event.module": "iis",
+        "fileset.name": "access",
+        "http.request.method": "GET",
+        "http.request.referrer": "-",
+        "http.response.status_code": 404,
+        "iis.access.sub_status": 0,
+        "iis.access.win32_status": 64,
+        "input.type": "log",
+        "log.offset": 702,
+        "service.type": "iis",
+        "source.address": "10.50.6.188",
+        "source.ip": "10.50.6.188",
+        "url.path": "/pbserver/..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153..\u00c1\u0153../winnt/system32/cmd.exe",
+        "url.query": "/c+dir+c:\\+/OG",
+        "user.name": "-",
+        "user_agent.device.name": "Other",
+        "user_agent.name": "IE",
+        "user_agent.original": "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)",
+        "user_agent.os.name": "Windows XP",
+        "user_agent.version": "8.0"
+    }
+]