Fix encoding errors #149

When the encoding of a database is not UTF8. Queries with special caracters might crash pg_activity with the message : UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe9 in position 26: ++ invalid continuation byte This patch fixes the issue by querying pg_database.encoding and using it to encode the string.
dalibo · Sep 23, 2021 · 35ea815 · 35ea815
1 parent 903fb87
commit 35ea815
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 115 deletions.
diff --git a/pgactivity/queries/get_blocking.sql b/pgactivity/queries/get_blocking.sql
@@ -2,7 +2,7 @@
 SELECT
       pid,
       application_name,
-      datname AS database,
+      sq.datname AS database,
       usename AS user,
       client,
       relation,
@@ -17,17 +17,19 @@ SELECT
       END AS state,
       CASE WHEN sq.query LIKE '<IDLE>%%'
           THEN NULL
-          ELSE sq.query
+          ELSE convert_from(sq.query::bytea, coalesce(pg_catalog.pg_encoding_to_char(b.encoding), 'UTF8'))
       END AS query,
       waiting AS wait
   FROM
       (
+      -- Transaction id lock
       SELECT
             blocking.pid,
             '<unknown>' AS application_name,
             pg_stat_activity.current_query AS query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -55,12 +57,14 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       UNION ALL
+      -- VirtualXid Lock
       SELECT
             blocking.pid,
             '<unknown>' AS application_name,
             pg_stat_activity.current_query AS query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -88,12 +92,14 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       UNION ALL
+      -- Relation Lock
       SELECT
             blocking.pid,
             '<unknown>' AS application_name,
             pg_stat_activity.current_query AS query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -125,18 +131,20 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       ) AS sq
+      LEFT OUTER JOIN pg_database b ON sq.datid = b.oid
 GROUP BY
       pid,
       application_name,
-      query,
+      database,
+      usename,
+      client,
+      relation,
       mode,
       locktype,
       duration,
-      datname,
-      usename,
-      client,
       state,
-      relation,
-      wait
+      query,
+      encoding,
+      wait_event
 ORDER BY
       duration DESC;
diff --git a/pgactivity/queries/get_blocking_post_90200.sql b/pgactivity/queries/get_blocking_post_90200.sql
@@ -2,24 +2,26 @@
 SELECT
       pid,
       application_name,
-      datname AS database,
+      sq.datname AS database,
       usename AS user,
       client,
       relation,
       mode,
       locktype AS type,
       duration,
       state,
-      query,
+      convert_from(sq.query::bytea, coalesce(pg_catalog.pg_encoding_to_char(b.encoding), 'UTF8')) AS query,
       waiting as wait
   FROM
       (
+      -- Transaction id lock
       SELECT
             blocking.pid,
             pg_stat_activity.application_name,
             pg_stat_activity.query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -48,12 +50,14 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       UNION ALL
+      -- VirtualXid Lock
       SELECT
             blocking.pid,
             pg_stat_activity.application_name,
             pg_stat_activity.query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -82,12 +86,14 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       UNION ALL
+      -- Relation Lock
       SELECT
             blocking.pid,
             pg_stat_activity.application_name,
             pg_stat_activity.query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -119,18 +125,20 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       ) AS sq
+      LEFT OUTER JOIN pg_database b ON sq.datid = b.oid
 GROUP BY
       pid,
       application_name,
-      query,
+      database,
+      usename,
+      client,
+      relation,
       mode,
       locktype,
       duration,
-      datname,
-      usename,
-      client,
       state,
-      relation,
-      wait
+      query,
+      encoding,
+      wait_event
 ORDER BY
       duration DESC;
diff --git a/pgactivity/queries/get_blocking_post_90600.sql b/pgactivity/queries/get_blocking_post_90600.sql
@@ -2,24 +2,26 @@
 SELECT
       pid,
       application_name,
-      datname AS database,
+      sq.datname AS database,
       usename AS user,
       client,
       relation,
       mode,
       locktype AS type,
       duration,
       state,
-      query,
+      convert_from(sq.query::bytea, coalesce(pg_catalog.pg_encoding_to_char(b.encoding), 'UTF8')) AS query,
       wait_event as wait
   FROM
       (
+      -- Transaction id lock
       SELECT
             blocking.pid,
             pg_stat_activity.application_name,
             pg_stat_activity.query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -48,12 +50,14 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       UNION ALL
+      -- VirtualXid Lock
       SELECT
             blocking.pid,
             pg_stat_activity.application_name,
             pg_stat_activity.query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -82,12 +86,14 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       UNION ALL
+      -- Relation Lock
       SELECT
             blocking.pid,
             pg_stat_activity.application_name,
             pg_stat_activity.query,
             blocking.mode,
             pg_stat_activity.datname,
+            pg_stat_activity.datid,
             pg_stat_activity.usename,
             CASE WHEN pg_stat_activity.client_addr IS NULL
                 THEN 'local'
@@ -119,18 +125,20 @@ SELECT
                 ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
             END
       ) AS sq
+      LEFT OUTER JOIN pg_database b ON sq.datid = b.oid
 GROUP BY
       pid,
       application_name,
-      query,
+      database,
+      usename,
+      client,
+      relation,
       mode,
       locktype,
       duration,
-      datname,
-      usename,
-      client,
       state,
-      relation,
-      wait
+      query,
+      encoding,
+      wait_event
 ORDER BY
       duration DESC;
diff --git a/pgactivity/queries/get_pg_activity.sql b/pgactivity/queries/get_pg_activity.sql
@@ -1,28 +1,29 @@
 -- Get data from pg_activity before pg 9.2
 SELECT
-      pg_stat_activity.procpid AS pid,
+      a.procpid AS pid,
       '<unknown>' AS application_name,
-      pg_stat_activity.datname AS database,
-      CASE WHEN pg_stat_activity.client_addr IS NULL
+      a.datname AS database,
+      CASE WHEN a.client_addr IS NULL
           THEN 'local'
-          ELSE pg_stat_activity.client_addr::TEXT
+          ELSE a.client_addr::TEXT
       END AS client,
-      EXTRACT(epoch FROM (NOW() - pg_stat_activity.{duration_column})) AS duration,
-      pg_stat_activity.waiting AS wait,
-      pg_stat_activity.usename AS user,
+      EXTRACT(epoch FROM (NOW() - a.{duration_column})) AS duration,
+      a.waiting AS wait,
+      a.usename AS user,
       CASE
-          WHEN pg_stat_activity.current_query = '<IDLE> in transaction (aborted)' THEN 'idle in transaction (aborted)'
-          WHEN pg_stat_activity.current_query = '<IDLE> in transaction' THEN 'idle in transaction'
-          WHEN pg_stat_activity.current_query = '<IDLE>' THEN 'idle'
+          WHEN a.current_query = '<IDLE> in transaction (aborted)' THEN 'idle in transaction (aborted)'
+          WHEN a.current_query = '<IDLE> in transaction' THEN 'idle in transaction'
+          WHEN a.current_query = '<IDLE>' THEN 'idle'
           ELSE 'active'
       END AS state,
       CASE
-          WHEN pg_stat_activity.current_query LIKE '<IDLE>%%' THEN NULL
-          ELSE pg_stat_activity.current_query
+          WHEN a.current_query LIKE '<IDLE>%%' THEN NULL
+          ELSE convert_from(a.current_query::bytea, coalesce(pg_catalog.pg_encoding_to_char(b.encoding), 'UTF8'))
       END AS query,
       false AS is_parallel_worker
   FROM
-        pg_stat_activity
+        pg_stat_activity a
+        LEFT OUTER JOIN pg_database b ON a.datid = b.oid
  WHERE
       current_query <> '<IDLE>'
   AND procpid <> pg_backend_pid()
@@ -31,4 +32,4 @@ SELECT
           ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
       END
 ORDER BY
-      EXTRACT(epoch FROM (NOW() - pg_stat_activity.{duration_column})) DESC
+      EXTRACT(epoch FROM (NOW() - a.{duration_column})) DESC
diff --git a/pgactivity/queries/get_pg_activity_post_100000.sql b/pgactivity/queries/get_pg_activity_post_100000.sql
@@ -1,23 +1,24 @@
 -- Get data from pg_activity from pg 10 to pg 11
 -- We assume a background_worker with a not null query is a parallel worker.
 SELECT
-      pg_stat_activity.pid AS pid,
-      pg_stat_activity.application_name AS application_name,
-      pg_stat_activity.datname AS database,
-      CASE WHEN pg_stat_activity.client_addr IS NULL
+      a.pid AS pid,
+      a.application_name AS application_name,
+      a.datname AS database,
+      CASE WHEN a.client_addr IS NULL
           THEN 'local'
-          ELSE pg_stat_activity.client_addr::TEXT
+          ELSE a.client_addr::TEXT
       END AS client,
-      EXTRACT(epoch FROM (NOW() - pg_stat_activity.{duration_column})) AS duration,
-      pg_stat_activity.wait_event as wait,
-      pg_stat_activity.usename AS user,
-      pg_stat_activity.state AS state,
-      pg_stat_activity.query AS query,
-      (   pg_stat_activity.backend_type = 'background worker'
-          AND pg_stat_activity.query IS NOT NULL
+      EXTRACT(epoch FROM (NOW() - a.{duration_column})) AS duration,
+      a.wait_event as wait,
+      a.usename AS user,
+      a.state AS state,
+      convert_from(a.query::bytea, coalesce(pg_catalog.pg_encoding_to_char(b.encoding), 'UTF8')) AS query,
+      (   a.backend_type = 'background worker'
+          AND a.query IS NOT NULL
       ) AS is_parallel_worker
   FROM
-      pg_stat_activity
+      pg_stat_activity a
+      LEFT OUTER JOIN pg_database b ON a.datid = b.oid
  WHERE
       state <> 'idle'
   AND pid <> pg_backend_pid()
@@ -26,4 +27,4 @@ SELECT
           ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
       END
 ORDER BY
-      EXTRACT(epoch FROM (NOW() - pg_stat_activity.{duration_column})) DESC;
+      EXTRACT(epoch FROM (NOW() - a.{duration_column})) DESC;
diff --git a/pgactivity/queries/get_pg_activity_post_110000.sql b/pgactivity/queries/get_pg_activity_post_110000.sql
@@ -1,26 +1,27 @@
 -- Get data from pg_activity since pg 11
 SELECT
-      pg_stat_activity.pid AS pid,
-      pg_stat_activity.application_name AS application_name,
-      pg_stat_activity.datname AS database,
-      CASE WHEN pg_stat_activity.client_addr IS NULL
+      a.pid AS pid,
+      a.application_name AS application_name,
+      a.datname AS database,
+      CASE WHEN a.client_addr IS NULL
           THEN 'local'
-          ELSE pg_stat_activity.client_addr::TEXT
+          ELSE a.client_addr::TEXT
       END AS client,
-      EXTRACT(epoch FROM (NOW() - pg_stat_activity.{duration_column})) AS duration,
-      pg_stat_activity.wait_event AS wait,
-      pg_stat_activity.usename AS user,
-      pg_stat_activity.state AS state,
-      pg_stat_activity.query AS query,
-      pg_stat_activity.backend_type = 'parallel worker' AS is_parallel_worker
+      EXTRACT(epoch FROM (NOW() - a.{duration_column})) AS duration,
+      a.wait_event AS wait,
+      a.usename AS user,
+      a.state AS state,
+      convert_from(a.query::bytea, coalesce(pg_catalog.pg_encoding_to_char(b.encoding), 'UTF8')) AS query,
+      a.backend_type = 'parallel worker' AS is_parallel_worker
  FROM
-      pg_stat_activity
+      pg_stat_activity a
+      LEFT OUTER JOIN pg_database b ON a.datid = b.oid
  WHERE
-      state <> 'idle'
-  AND pid <> pg_backend_pid()
+      a.state <> 'idle'
+  AND a.pid <> pg_catalog.pg_backend_pid()
   AND CASE WHEN %(min_duration)s = 0
           THEN true
           ELSE extract(epoch from now() - {duration_column}) > %(min_duration)s
       END
 ORDER BY
-      EXTRACT(epoch FROM (NOW() - pg_stat_activity.{duration_column})) DESC;
+      EXTRACT(epoch FROM (NOW() - a.{duration_column})) DESC;