Complete the migration to event.duration for mysql.slowlog

elastic · Jan 23, 2019 · d975b1f · d975b1f
1 parent c3e592a
commit d975b1f
Show file tree

Hide file tree

Showing 10 changed files with 17 additions and 31 deletions.
diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml
@@ -627,6 +627,11 @@
   to: mysql.thread_id
   alias: true
 
+- from: mysql.slowlog.query_time.sec
+  to: event.duration
+  alias: false
+  scale: 1000000000
+
 ## NGINX module
 
 - from: nginx.access.user_name

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -6327,16 +6327,6 @@ Contains fields from the MySQL slow logs.
 
 
 
-*`mysql.slowlog.query_time.sec`*::
-+
---
-type: float
-
-The total time the query took, in seconds, as a floating point number.
-
-
---
-
 *`mysql.slowlog.lock_time.sec`*::
 +
 --

diff --git a/filebeat/module/mysql/fields.go b/filebeat/module/mysql/fields.go
diff --git a/filebeat/module/mysql/slowlog/_meta/fields.yml b/filebeat/module/mysql/slowlog/_meta/fields.yml
@@ -3,10 +3,6 @@
   description: >
     Contains fields from the MySQL slow logs.
   fields:
-    - name: query_time.sec
-      type: float
-      description: >
-        The total time the query took, in seconds, as a floating point number.
     - name: lock_time.sec
       type: float
       description: >
@@ -67,7 +63,7 @@
         Whether the query needed temporary tables on disk.
     - name: tmp_tables
       type: long
-      description: > 
+      description: >
         Number of temporary tables created for this query
     - name: tmp_disk_tables
       type: long
@@ -76,7 +72,7 @@
     - name: tmp_table_sizes
       type: long
       format: bytes
-      description: 
+      description:
         Size of temporary tables created for this query.
     - name: filesort
       type: boolean

diff --git a/filebeat/module/mysql/slowlog/ingest/pipeline.json b/filebeat/module/mysql/slowlog/ingest/pipeline.json
@@ -4,7 +4,7 @@
     "grok": {
       "field": "message",
       "patterns":[
-        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:mysql.slowlog.query_time.sec:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
+        "^# User@Host: %{USER:user.name}(\\[%{USER:mysql.slowlog.current_user}\\])? @ %{HOSTNAME:source.domain}? \\[%{IP:source.ip}?\\]%{METRICSPACE}(Id:%{SPACE}%{NUMBER:mysql.thread_id:long}%{METRICSPACE})?(Thread_id:%{SPACE}%{NUMBER:mysql.thread_id}%{METRICSPACE})?(Schema:%{SPACE}%{WORD:mysql.slowlog.schema}?%{METRICSPACE})?(Last_errno: %{NUMBER:mysql.slowlog.last_errno:long}%{METRICSPACE})?(Killed: %{NUMBER:mysql.slowlog.killed:long}%{METRICSPACE})?(QC_hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Query_time: %{NUMBER:temp.duration:float}%{METRICSPACE})?(Lock_time: %{NUMBER:mysql.slowlog.lock_time.sec:float}%{METRICSPACE})?(Rows_sent: %{NUMBER:mysql.slowlog.rows_sent:long}%{METRICSPACE})?(Rows_examined: %{NUMBER:mysql.slowlog.rows_examined:long}%{METRICSPACE})?(Rows_affected: %{NUMBER:mysql.slowlog.rows_affected:long}%{METRICSPACE})?(Bytes_sent: %{NUMBER:mysql.slowlog.bytes_sent:long}%{METRICSPACE})?(Tmp_tables: %{NUMBER:mysql.slowlog.tmp_tables:long}%{METRICSPACE})?(Tmp_disk_tables: %{NUMBER:mysql.slowlog.tmp_disk_tables}%{METRICSPACE})?(Tmp_table_sizes: %{NUMBER:mysql.slowlog.tmp_table_sizes:long}%{METRICSPACE})?(InnoDB_trx_id: %{WORD:mysql.slowlog.innodb.trx_id}%{METRICSPACE})?(QC_Hit: %{WORD:mysql.slowlog.query_cache_hit}%{METRICSPACE})?(Full_scan: %{WORD:mysql.slowlog.full_scan}%{METRICSPACE})?(Full_join: %{WORD:mysql.slowlog.full_join}%{METRICSPACE})?(Tmp_table: %{WORD:mysql.slowlog.tmp_table}%{METRICSPACE})?(Tmp_table_on_disk: %{WORD:mysql.slowlog.tmp_table_on_disk}%{METRICSPACE})?(Filesort: %{WORD:mysql.slowlog.filesort}%{METRICSPACE})?(Filesort_on_disk: %{WORD:mysql.slowlog.filesort_on_disk}%{METRICSPACE})?(Merge_passes: %{NUMBER:mysql.slowlog.merge_passes:long}%{METRICSPACE})?(Priority_queue: %{WORD:mysql.slowlog.priority_queue}%{METRICSPACE})?(No InnoDB statistics available for this query%{METRICSPACE})?(InnoDB_IO_r_ops: %{NUMBER:mysql.slowlog.innodb.io_r_ops:long}%{METRICSPACE})?(InnoDB_IO_r_bytes: %{NUMBER:mysql.slowlog.innodb.io_r_bytes:long}%{METRICSPACE})?(InnoDB_IO_r_wait: %{NUMBER:mysql.slowlog.innodb.io_r_wait.sec:float}%{METRICSPACE})?(InnoDB_rec_lock_wait: %{NUMBER:mysql.slowlog.innodb.rec_lock_wait.sec:float}%{METRICSPACE})?(InnoDB_queue_wait: %{NUMBER:mysql.slowlog.innodb.queue_wait.sec:float}%{METRICSPACE})?(InnoDB_pages_distinct: %{NUMBER:mysql.slowlog.innodb.pages_distinct:long}%{METRICSPACE})?(Log_slow_rate_type: %{WORD:mysql.slowlog.log_slow_rate_type}%{METRICSPACE})?(Log_slow_rate_limit: %{NUMBER:mysql.slowlog.log_slow_rate_limit:long}%{METRICSPACE})?%{EXPLAIN}?(use %{WORD:mysql.slowlog.schema};\n)?SET timestamp=%{NUMBER:mysql.slowlog.timestamp:long};\n%{GREEDYMULTILINE:mysql.slowlog.query}"
         ],
       "pattern_definitions" : {
         "GREEDYMULTILINE": "(.|\n)*",
@@ -39,10 +39,15 @@
       }
     }
   }, {
-    "script":{
+    "script": {
       "lang": "painless",
-      "source": "ctx.event.duration = Math.round(ctx.mysql.slowlog.query_time.sec * 1000000) * 1000",
-      "if": "ctx.mysql.slowlog.query_time?.sec != null"
+      "source": "ctx.event.duration = Math.round(ctx.temp.duration * 1000000) * 1000",
+      "if": "ctx.temp?.duration != null"
+    }
+  }, {
+    "remove": {
+      "field": "temp.duration",
+      "ignore_missing": true
     }
   }, {
     "date": {

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.1.21.log-expected.json
@@ -15,7 +15,6 @@
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(2);",
         "mysql.slowlog.query_cache_hit": false,
-        "mysql.slowlog.query_time.sec": 2.000652,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
         "mysql.thread_id": "5",

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-10.2.12.log-expected.json
@@ -15,7 +15,6 @@
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(2)\nAS foo;",
         "mysql.slowlog.query_cache_hit": false,
-        "mysql.slowlog.query_time.sec": 2.000227,
         "mysql.slowlog.rows_affected": 0,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
@@ -46,7 +45,6 @@
         "mysql.slowlog.priority_queue": false,
         "mysql.slowlog.query": "call PROC('blah');",
         "mysql.slowlog.query_cache_hit": false,
-        "mysql.slowlog.query_time.sec": 178.30602,
         "mysql.slowlog.rows_affected": 3062,
         "mysql.slowlog.rows_examined": 53022772,
         "mysql.slowlog.rows_sent": 0,

diff --git a/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json b/filebeat/module/mysql/slowlog/test/mariadb-explain.log-expected.json
@@ -15,7 +15,6 @@
         "mysql.slowlog.lock_time.sec": 0.000337,
         "mysql.slowlog.query": "select count(*) from customer, orders, nation\n  where c_custkey=o_custkey\n    and c_nationkey=n_nationkey\n    and n_name='GERMANY';",
         "mysql.slowlog.query_cache_hit": false,
-        "mysql.slowlog.query_time.sec": 5.524103,
         "mysql.slowlog.rows_examined": 65633,
         "mysql.slowlog.rows_sent": 1,
         "mysql.slowlog.schema": "dbt3sf1",

diff --git a/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json b/filebeat/module/mysql/slowlog/test/mysql-5.7.22.log-expected.json
@@ -14,7 +14,6 @@
         "mysql.slowlog.current_user": "root",
         "mysql.slowlog.lock_time.sec": 0.0,
         "mysql.slowlog.query": "select sleep(15);",
-        "mysql.slowlog.query_time.sec": 15.000223,
         "mysql.slowlog.rows_examined": 0,
         "mysql.slowlog.rows_sent": 1,
         "mysql.thread_id": 7234,
@@ -37,7 +36,6 @@
         "mysql.slowlog.current_user": "debian-sys-maint",
         "mysql.slowlog.lock_time.sec": 6.1e-05,
         "mysql.slowlog.query": "SELECT count(*) FROM mysql.user WHERE user='root' and password='';",
-        "mysql.slowlog.query_time.sec": 0.000153,
         "mysql.slowlog.rows_examined": 5,
         "mysql.slowlog.rows_sent": 1,
         "service.type": "mysql",
@@ -59,7 +57,6 @@
         "mysql.slowlog.current_user": "appuser",
         "mysql.slowlog.lock_time.sec": 0.000212,
         "mysql.slowlog.query": "SELECT mcu.mcu_guid, mcu.cus_guid, mcu.mcu_url, mcu.mcu_crawlelements, mcu.mcu_order, GROUP_CONCAT(mca.mca_guid SEPARATOR \";\") as mca_guid\n                    FROM kat_mailcustomerurl mcu, kat_customer cus, kat_mailcampaign mca\n                    WHERE cus.cus_guid = mcu.cus_guid\n                        AND cus.pro_code = 'CYB'\n                        AND cus.cus_offline = 0\n                        AND mca.cus_guid = cus.cus_guid\n                        AND (mcu.mcu_date IS NULL OR mcu.mcu_date < CURDATE())\n                        AND mcu.mcu_crawlelements IS NOT NULL\n                    GROUP BY mcu.mcu_guid\n                    ORDER BY mcu.mcu_order ASC\n                    LIMIT 1000;",
-        "mysql.slowlog.query_time.sec": 4.071491,
         "mysql.slowlog.rows_examined": 1489615,
         "mysql.slowlog.rows_sent": 1000,
         "mysql.thread_id": 10997316,
@@ -83,7 +80,6 @@
         "mysql.slowlog.current_user": "appuser",
         "mysql.slowlog.lock_time.sec": 3.6e-05,
         "mysql.slowlog.query": "call load_stats(1, '2017-04-28 00:00:00');",
-        "mysql.slowlog.query_time.sec": 10.346539,
         "mysql.slowlog.rows_examined": 4751313,
         "mysql.slowlog.rows_sent": 0,
         "mysql.thread_id": 10999834,

diff --git a/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json b/filebeat/module/mysql/slowlog/test/percona-ubuntu-5.7.19-innodb.log-expected.json
@@ -25,7 +25,6 @@
         "mysql.slowlog.merge_passes": 0,
         "mysql.slowlog.query": "SELECT t.table_schema, t.table_name, column_name, `auto_increment`,\n                  pow(2, case data_type\n                    when 'tinyint'   then 7\n                    when 'smallint'  then 15\n                    when 'mediumint' then 23\n                    when 'int'       then 31\n                    when 'bigint'    then 63\n                    end+(column_type like '% unsigned'))-1 as max_int\n                  FROM information_schema.tables t\n                  JOIN information_schema.columns c\n                    ON BINARY t.table_schema = c.table_schema AND BINARY t.table_name = c.table_name\n                  WHERE c.extra = 'auto_increment' AND t.auto_increment IS NOT NULL;",
         "mysql.slowlog.query_cache_hit": false,
-        "mysql.slowlog.query_time.sec": 0.050365,
         "mysql.slowlog.rows_affected": 0,
         "mysql.slowlog.rows_examined": 3146,
         "mysql.slowlog.rows_sent": 16,
@@ -72,7 +71,6 @@
         "mysql.slowlog.merge_passes": 0,
         "mysql.slowlog.query": "UPDATE test    SET test.state = 'NOT_RELEVANT', modified = now()  WHERE test.id IN (26328833, 390, 149386, 152268, 160997, 165304, 168524, 184105, 193022, 194533, 194862, 196469, 196487, 246398, 256594, 260566, 261862, 262342, 263701, 264166, 264607, 267671, 274879, 276704, 280964, 284366, 289323, 289843, 290004, 298999, 301213, 303494, 307920, 311905, 316311, 318404, 330846, 340751, 341433, 357191, 369184, 376876, 378360, 378492, 379470, 382131, 384077, 388368, 396815, 396881, 398272, 398950, 399589, 401299, 408787, 411293, 419109, 425953, 427659, 433183, 437030, 438332, 438386, 447037, 454231, 455257, 455344, 456385, 460420, 460425, 461252, 462338, 462531, 462684, 463104, 463395, 471073, 480069, 480078, 482399, 485205, 487971, 497191, 500261, 501855, 517585, 519310, 519654, 522575, 538425, 543560, 562315, 573934, 583466, 583490, 583502, 597605, 600875, 601546, 603879, 604467, 604619, 757786, 797285, 799155, 802905, 806268, 806798, 811974, 819684, 822629, 826406, 837733, 840128, 840131, 840251, 840277, 840302, 842966, 844294, 844300, 847837, 852503, 854272, 854299, 862983, 881405, 881461, 881467, 881560, 881908, 882435, 882453, 882651, 882711, 882811, 888265, 888286, 914091, 916288, 916316, 917708, 918238, 918887, 919222, 926607, 976977, 977010, 977067, 977131, 977185, 988249, 988276, 988336, 988360, 988504, 990994);",
         "mysql.slowlog.query_cache_hit": false,
-        "mysql.slowlog.query_time.sec": 153.88348,
         "mysql.slowlog.rows_affected": 19198,
         "mysql.slowlog.rows_examined": 120309968,
         "mysql.slowlog.rows_sent": 0,