diff --git a/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql b/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql new file mode 100644 index 00000000000..6e1440b9194 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/monitoring/bigeye_usage/view.sql @@ -0,0 +1,21 @@ +SELECT + submission_date, + reference_project_id, + reference_dataset_id, + reference_table_id, + creation_date, + task_duration, + total_terabytes_processed, + total_terabytes_billed, + total_slot_ms, + cost, + job_id, + user_email AS service_account, + bigeye_metric_id, + bigeye_monitored_table_id, +FROM + `moz-fx-data-shared-prod.monitoring.bigquery_usage` +WHERE + DATE(submission_date) = "2024-09-24" + AND user_type = "bigeye" + AND total_slot_ms IS NOT NULL diff --git a/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql b/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql index e3414854032..923ff134977 100644 --- a/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql +++ b/sql/moz-fx-data-shared-prod/monitoring/bigquery_usage/view.sql @@ -18,6 +18,8 @@ SELECT THEN "search-terms" WHEN user_email LIKE "%airflow%" THEN "airflow" + WHEN user_email LIKE "%bigeye%" + THEN "bigeye" WHEN ENDS_WITH(user_email, "mozilla.com") THEN "individual" WHEN ENDS_WITH(user_email, "mozillafoundation.org") diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql index 1dfd86230dd..f7ec3adc372 100644 --- a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql +++ b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/query.sql @@ -48,6 +48,8 @@ jobs_by_project AS ( user_email, REGEXP_EXTRACT(query, r'Username: (.*?),') AS username, REGEXP_EXTRACT(query, r'Query ID: (\w+), ') AS query_id, + REGEXP_EXTRACT(query, r'\'metric_id:(\d+)\'') AS bigeye_metric_id, + REPLACE(REGEXP_EXTRACT(LOWER(query), r'`__grain_start__` from `([\w\d_\-\.`]+) as'), "`", "") AS bigeye_monitored_table_id, FROM `{{project}}.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` AS jp LEFT JOIN @@ -103,6 +105,8 @@ SELECT DISTINCT jo.error_reason, jo.error_message, jo.resource_warning, + jo.bigeye_metric_id, + jo.bigeye_monitored_table_id, @submission_date AS submission_date, FROM jobs_by_org AS jo diff --git a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml index 25806bd6656..bfcc15dd802 100644 --- a/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml +++ b/sql/moz-fx-data-shared-prod/monitoring_derived/bigquery_usage_v2/schema.yaml @@ -125,6 +125,16 @@ fields: type: STRING description: The warning message that appears if the resource usage is above the internal threshold of the system +- mode: NULLABLE + name: bigeye_metric_id + type: STRING + description: Metric ID corresponding to the metric executed by BigEye (Data Observability Platform). + +- mode: NULLABLE + name: bigeye_monitored_table_id + type: STRING + description: Fully qualified table id of the table BigEye run a monitor against (volume and freshness). + - mode: NULLABLE name: submission_date type: DATE