From 24650d4c45eb437ec71b76f00c2d4da00747bc31 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 25 Sep 2024 15:50:05 +0200 Subject: [PATCH 1/2] feat: add monitoring bigeye_usage view to allow us to get more insight into bq usage and costs from bigeye --- .../monitoring/bigeye_usage/view.sql | 20 +++++++++++++++++++ .../monitoring/bigquery_usage/view.sql | 2 ++ .../bigquery_usage_v2/query.sql | 2 ++ .../bigquery_usage_v2/schema.yaml | 5 +++++ 4 files changed, 29 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql diff --git a/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql b/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql new file mode 100644 index 00000000000..fc857e30fd3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql @@ -0,0 +1,20 @@ +SELECT + submission_date, + reference_project_id, + reference_dataset_id, + reference_table_id, + creation_date, + task_duration, + total_terabytes_processed, + total_terabytes_billed, + total_slot_ms, + cost, + job_id, + user_email AS service_account, + bigeye_metric_id, +FROM + `moz-fx-data-shared-prod.monitoring.bigquery_usage` +WHERE + DATE(submission_date) = "2024-09-24" + AND user_type = "bigeye" + AND total_slot_ms IS NOT NULL diff --git a/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql b/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql index e3414854032..923ff134977 100644 --- a/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql +++ b/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql @@ -18,6 +18,8 @@ SELECT THEN "search-terms" WHEN user_email LIKE "%airflow%" THEN "airflow" + WHEN user_email LIKE "%bigeye%" + THEN "bigeye" WHEN ENDS_WITH(user_email, "mozilla.com") THEN "individual" WHEN ENDS_WITH(user_email, "mozillafoundation.org") diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql index 1dfd86230dd..441ec8f5d5a 100644 --- a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql @@ -48,6 +48,7 @@ jobs_by_project AS ( user_email, REGEXP_EXTRACT(query, r'Username: (.*?),') AS username, REGEXP_EXTRACT(query, r'Query ID: (\w+), ') AS query_id, + REGEXP_EXTRACT(query, r'\'metric_id:(\d+)\'') AS bigeye_metric_id, FROM `{{project}}.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` AS jp LEFT JOIN @@ -103,6 +104,7 @@ SELECT DISTINCT jo.error_reason, jo.error_message, jo.resource_warning, + jo.bigeye_metric_id, @submission_date AS submission_date, FROM jobs_by_org AS jo diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml index 25806bd6656..50789a743ec 100644 --- a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml +++ b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml @@ -125,6 +125,11 @@ fields: type: STRING description: The warning message that appears if the resource usage is above the internal threshold of the system +- mode: NULLABLE + name: bigeye_metric_id + type: STRING + description: Metric ID corresponding to the metric executed by BigEye (Data Observability Platform). + - mode: NULLABLE name: submission_date type: DATE From 477a7529893bb796e55691da6be3916ed9d3a708 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 25 Sep 2024 17:15:54 +0200 Subject: [PATCH 2/2] feat: add extract of bigeye table id checked in the bq information_schema --- sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql | 1 + .../monitoring_derived/bigquery_usage_v2/query.sql | 2 ++ .../monitoring_derived/bigquery_usage_v2/schema.yaml | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql b/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql index fc857e30fd3..6e1440b9194 100644 --- a/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql +++ b/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql @@ -12,6 +12,7 @@ SELECT job_id, user_email AS service_account, bigeye_metric_id, + bigeye_monitored_table_id, FROM `moz-fx-data-shared-prod.monitoring.bigquery_usage` WHERE diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql index 441ec8f5d5a..f7ec3adc372 100644 --- a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql @@ -49,6 +49,7 @@ jobs_by_project AS ( REGEXP_EXTRACT(query, r'Username: (.*?),') AS username, REGEXP_EXTRACT(query, r'Query ID: (\w+), ') AS query_id, REGEXP_EXTRACT(query, r'\'metric_id:(\d+)\'') AS bigeye_metric_id, + REPLACE(REGEXP_EXTRACT(LOWER(query), r'`__grain_start__` from `([\w\d_\-\.`]+) as'), "`", "") AS bigeye_monitored_table_id, FROM `{{project}}.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` AS jp LEFT JOIN @@ -105,6 +106,7 @@ SELECT DISTINCT jo.error_message, jo.resource_warning, jo.bigeye_metric_id, + jo.bigeye_monitored_table_id, @submission_date AS submission_date, FROM jobs_by_org AS jo diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml index 50789a743ec..bfcc15dd802 100644 --- a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml +++ b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml @@ -130,6 +130,11 @@ fields: type: STRING description: Metric ID corresponding to the metric executed by BigEye (Data Observability Platform). +- mode: NULLABLE + name: bigeye_monitored_table_id + type: STRING + description: Fully qualified table id of the table BigEye run a monitor against (volume and freshness). + - mode: NULLABLE name: submission_date type: DATE