From c42866fa746940ea8d4e6f43f70f36e1e7e01bd9 Mon Sep 17 00:00:00 2001 From: Jeff Klukas Date: Wed, 30 Oct 2019 14:01:17 -0400 Subject: [PATCH] Remove smoot_usage v1 queries GUD now uses the v2 tables and the v1 tables are no longer needed. See also https://github.com/mozilla/telemetry-airflow/pull/666 --- .../smoot_usage_all_mtr_v1/query.sql | 4 - sql/telemetry/smoot_usage_all_v1/view.sql | 21 -- .../smoot_usage_desktop_raw_v1/query.sql | 114 ---------- sql/telemetry/smoot_usage_desktop_v1/view.sql | 76 ------- .../smoot_usage_fxa_raw_v1/query.sql | 95 -------- sql/telemetry/smoot_usage_fxa_v1/view.sql | 76 ------- .../smoot_usage_nondesktop_raw_v1/query.sql | 99 --------- .../smoot_usage_nondesktop_v1/view.sql | 91 -------- .../smoot_usage_all_mtr_v1/query.sql | 4 - .../telemetry/smoot_usage_all_v1/view.sql | 21 -- .../smoot_usage_desktop_raw_v1/query.sql | 101 --------- .../telemetry/smoot_usage_desktop_v1/view.sql | 76 ------- .../smoot_usage_fxa_raw_v1/query.sql | 82 ------- .../telemetry/smoot_usage_fxa_v1/view.sql | 76 ------- .../smoot_usage_nondesktop_raw_v1/query.sql | 86 -------- .../smoot_usage_nondesktop_v1/view.sql | 91 -------- templates/telemetry/smoot_usage_raw.sql.py | 204 ------------------ 17 files changed, 1317 deletions(-) delete mode 100644 sql/telemetry/smoot_usage_all_mtr_v1/query.sql delete mode 100644 sql/telemetry/smoot_usage_all_v1/view.sql delete mode 100644 sql/telemetry/smoot_usage_desktop_raw_v1/query.sql delete mode 100644 sql/telemetry/smoot_usage_desktop_v1/view.sql delete mode 100644 sql/telemetry/smoot_usage_fxa_raw_v1/query.sql delete mode 100644 sql/telemetry/smoot_usage_fxa_v1/view.sql delete mode 100644 sql/telemetry/smoot_usage_nondesktop_raw_v1/query.sql delete mode 100644 sql/telemetry/smoot_usage_nondesktop_v1/view.sql delete mode 100644 templates/telemetry/smoot_usage_all_mtr_v1/query.sql delete mode 100644 templates/telemetry/smoot_usage_all_v1/view.sql delete mode 100644 templates/telemetry/smoot_usage_desktop_raw_v1/query.sql delete mode 100644 templates/telemetry/smoot_usage_desktop_v1/view.sql delete mode 100644 templates/telemetry/smoot_usage_fxa_raw_v1/query.sql delete mode 100644 templates/telemetry/smoot_usage_fxa_v1/view.sql delete mode 100644 templates/telemetry/smoot_usage_nondesktop_raw_v1/query.sql delete mode 100644 templates/telemetry/smoot_usage_nondesktop_v1/view.sql delete mode 100755 templates/telemetry/smoot_usage_raw.sql.py diff --git a/sql/telemetry/smoot_usage_all_mtr_v1/query.sql b/sql/telemetry/smoot_usage_all_mtr_v1/query.sql deleted file mode 100644 index 1b1a1883bd3..00000000000 --- a/sql/telemetry/smoot_usage_all_mtr_v1/query.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT - * -FROM - smoot_usage_all_v1 diff --git a/sql/telemetry/smoot_usage_all_v1/view.sql b/sql/telemetry/smoot_usage_all_v1/view.sql deleted file mode 100644 index e0684160671..00000000000 --- a/sql/telemetry/smoot_usage_all_v1/view.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_all_v1` -AS -SELECT - * -FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_desktop_v1` --- -UNION ALL --- -SELECT - * -FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_nondesktop_v1` --- -UNION ALL --- -SELECT - * -FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_fxa_v1` diff --git a/sql/telemetry/smoot_usage_desktop_raw_v1/query.sql b/sql/telemetry/smoot_usage_desktop_raw_v1/query.sql deleted file mode 100644 index c0b225fc5c6..00000000000 --- a/sql/telemetry/smoot_usage_desktop_raw_v1/query.sql +++ /dev/null @@ -1,114 +0,0 @@ -CREATE TEMP FUNCTION - udf_bitmask_lowest_7() AS (0x7F); -CREATE TEMP FUNCTION - udf_active_n_weeks_ago(x INT64, n INT64) - RETURNS BOOLEAN - AS ( - BIT_COUNT(x >> (7 * n) & udf_bitmask_lowest_7()) > 0 - ); -CREATE TEMP FUNCTION - udf_bitcount_lowest_7(x INT64) AS ( - BIT_COUNT(x & udf_bitmask_lowest_7()) - ); --- --- Query generated by: templates/smoot_usage_raw.sql.py --source clients_last_seen_v1 - -WITH - base AS ( - SELECT - * REPLACE(normalized_channel AS channel) - FROM - clients_last_seen_v1) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ -- - STRUCT('Any Firefox Desktop Activity' AS usage, - STRUCT( - COUNTIF(days_since_seen < 1) AS dau, - COUNTIF(days_since_seen < 7) AS wau, - COUNTIF(days_since_seen < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_seen_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_seen_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 1) AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('Firefox Desktop Visited 5 URI' AS usage, - STRUCT( - COUNTIF(days_since_visited_5_uri < 1) AS dau, - COUNTIF(days_since_visited_5_uri < 7) AS wau, - COUNTIF(days_since_visited_5_uri < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_visited_5_uri_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_visited_5_uri_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 1) AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('Firefox Desktop Opened Dev Tools' AS usage, - STRUCT( - COUNTIF(days_since_opened_dev_tools < 1) AS dau, - COUNTIF(days_since_opened_dev_tools < 7) AS wau, - COUNTIF(days_since_opened_dev_tools < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_opened_dev_tools_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_opened_dev_tools_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 1) AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('New Firefox Desktop Profile Created' AS usage, - STRUCT( - COUNTIF(days_since_created_profile < 1) AS dau, - NULL AS wau, - NULL AS mau, - NULL AS active_days_in_week - ) AS metrics_daily, - STRUCT( - NULL AS active_in_week_0, - NULL AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - NULL AS active_in_week_1, - NULL AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile) - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel diff --git a/sql/telemetry/smoot_usage_desktop_v1/view.sql b/sql/telemetry/smoot_usage_desktop_v1/view.sql deleted file mode 100644 index 62285763008..00000000000 --- a/sql/telemetry/smoot_usage_desktop_v1/view.sql +++ /dev/null @@ -1,76 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_desktop_v1` AS -WITH - base AS ( - SELECT - * - FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_desktop_raw_v1`), - -- - daily AS ( - SELECT - submission_date AS `date`, - metrics_daily.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week1 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 6 day) AS `date`, - metrics_1_week_post_new_profile.*, - * - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week2 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 13 day) AS `date`, - metrics_2_week_post_new_profile.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - joined AS ( - SELECT - * EXCEPT (submission_date, - metrics, - metrics_daily, - metrics_1_week_post_new_profile, - metrics_2_week_post_new_profile) - FROM - daily - FULL JOIN - new_profile_week1 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) - FULL JOIN - new_profile_week2 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) ) - -- -SELECT - * -FROM - joined diff --git a/sql/telemetry/smoot_usage_fxa_raw_v1/query.sql b/sql/telemetry/smoot_usage_fxa_raw_v1/query.sql deleted file mode 100644 index 7bc6216fb96..00000000000 --- a/sql/telemetry/smoot_usage_fxa_raw_v1/query.sql +++ /dev/null @@ -1,95 +0,0 @@ -CREATE TEMP FUNCTION - udf_bitmask_lowest_7() AS (0x7F); -CREATE TEMP FUNCTION - udf_active_n_weeks_ago(x INT64, n INT64) - RETURNS BOOLEAN - AS ( - BIT_COUNT(x >> (7 * n) & udf_bitmask_lowest_7()) > 0 - ); -CREATE TEMP FUNCTION - udf_bitcount_lowest_7(x INT64) AS ( - BIT_COUNT(x & udf_bitmask_lowest_7()) - ); --- --- Query generated by: templates/smoot_usage_raw.sql.py --source fxa_users_last_seen_v1 - -WITH - base AS ( - SELECT - * REPLACE(country_names.code AS country), - user_id AS client_id, - days_since_registered AS days_since_created_profile, - language AS locale, - os_name AS os, - CAST(NULL AS STRING) AS app_name, - CAST(NULL AS STRING) AS channel - FROM - fxa_users_last_seen_v1 fxa - LEFT JOIN - static.country_names_v1 country_names - ON - fxa.country = country_names.`name` - ) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ -- - STRUCT('Any Firefox Account Activity' AS usage, - STRUCT( - COUNTIF(days_since_seen < 1) AS dau, - COUNTIF(days_since_seen < 7) AS wau, - COUNTIF(days_since_seen < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_seen_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_seen_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 1) AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('New Firefox Account Registered' AS usage, - STRUCT( - COUNTIF(days_since_registered < 1) AS dau, - NULL AS wau, - NULL AS mau, - NULL AS active_days_in_week - ) AS metrics_daily, - STRUCT( - NULL AS active_in_week_0, - NULL AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - NULL AS active_in_week_1, - NULL AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile) - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel diff --git a/sql/telemetry/smoot_usage_fxa_v1/view.sql b/sql/telemetry/smoot_usage_fxa_v1/view.sql deleted file mode 100644 index c75a02e7f29..00000000000 --- a/sql/telemetry/smoot_usage_fxa_v1/view.sql +++ /dev/null @@ -1,76 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_fxa_v1` AS -WITH - base AS ( - SELECT - * - FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_fxa_raw_v1` ), - -- - daily AS ( - SELECT - submission_date AS `date`, - metrics_daily.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week1 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 6 day) AS `date`, - metrics_1_week_post_new_profile.*, - * - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week2 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 13 day) AS `date`, - metrics_2_week_post_new_profile.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - joined AS ( - SELECT - * EXCEPT (submission_date, - metrics, - metrics_daily, - metrics_1_week_post_new_profile, - metrics_2_week_post_new_profile) - FROM - daily - FULL JOIN - new_profile_week1 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) - FULL JOIN - new_profile_week2 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) ) - -- -SELECT - * -FROM - joined diff --git a/sql/telemetry/smoot_usage_nondesktop_raw_v1/query.sql b/sql/telemetry/smoot_usage_nondesktop_raw_v1/query.sql deleted file mode 100644 index f543f9fdee7..00000000000 --- a/sql/telemetry/smoot_usage_nondesktop_raw_v1/query.sql +++ /dev/null @@ -1,99 +0,0 @@ -CREATE TEMP FUNCTION - udf_bitmask_lowest_7() AS (0x7F); -CREATE TEMP FUNCTION - udf_active_n_weeks_ago(x INT64, n INT64) - RETURNS BOOLEAN - AS ( - BIT_COUNT(x >> (7 * n) & udf_bitmask_lowest_7()) > 0 - ); -CREATE TEMP FUNCTION - udf_bitcount_lowest_7(x INT64) AS ( - BIT_COUNT(x & udf_bitmask_lowest_7()) - ); --- --- Query generated by: templates/smoot_usage_raw.sql.py --source nondesktop_clients_last_seen_v1 - -WITH - base AS ( - SELECT - *, - normalized_channel AS channel - FROM - nondesktop_clients_last_seen_v1 - WHERE - -- We apply this filter here rather than in the live view because this field - -- is not normalized and there are many single pings that come in with unique - -- nonsensical app_name values. App names are documented in - -- https://docs.telemetry.mozilla.org/concepts/choosing_a_dataset_mobile.html#products-overview - (STARTS_WITH(app_name, 'FirefoxReality') OR app_name IN ( - 'Fenix', - 'Fennec', -- Firefox for Android and Firefox for iOS - 'Focus', - 'FirefoxConnect', -- Amazon Echo - 'FirefoxForFireTV', - 'Zerda')) -- Firefox Lite, previously called Rocket - -- There are also many strange nonsensical entries for os, so we filter here. - AND os IN ('Android', 'iOS')) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ -- - STRUCT('Any Firefox Non-desktop Activity' AS usage, - STRUCT( - COUNTIF(days_since_seen < 1) AS dau, - COUNTIF(days_since_seen < 7) AS wau, - COUNTIF(days_since_seen < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_seen_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_seen_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 1) AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('New Firefox Non-desktop Profile Created' AS usage, - STRUCT( - COUNTIF(days_since_created_profile < 1) AS dau, - NULL AS wau, - NULL AS mau, - NULL AS active_days_in_week - ) AS metrics_daily, - STRUCT( - NULL AS active_in_week_0, - NULL AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - NULL AS active_in_week_1, - NULL AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile) - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel diff --git a/sql/telemetry/smoot_usage_nondesktop_v1/view.sql b/sql/telemetry/smoot_usage_nondesktop_v1/view.sql deleted file mode 100644 index b3e4c9204ef..00000000000 --- a/sql/telemetry/smoot_usage_nondesktop_v1/view.sql +++ /dev/null @@ -1,91 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_nondesktop_v1` AS -WITH - base AS ( - SELECT - * - FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_nondesktop_raw_v1` ), - -- - daily AS ( - SELECT - submission_date AS `date`, - metrics_daily.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week1 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 6 day) AS `date`, - metrics_1_week_post_new_profile.*, - * - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week2 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 13 day) AS `date`, - metrics_2_week_post_new_profile.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - joined AS ( - SELECT - * EXCEPT (submission_date, - metrics, - metrics_daily, - metrics_1_week_post_new_profile, - metrics_2_week_post_new_profile) - REPLACE ( - CASE app_name - WHEN 'Fennec' THEN CONCAT(app_name, ' ', os) - WHEN 'Focus' THEN CONCAT(app_name, ' ', os) - WHEN 'Zerda' THEN 'Firefox Lite' - ELSE app_name - END AS app_name) - FROM - daily - FULL JOIN - new_profile_week1 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) - FULL JOIN - new_profile_week2 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) ) - -- -SELECT - * -FROM - joined - -- -UNION ALL - -- Also present each app as its own usage criterion. App names are documented in - -- https://docs.telemetry.mozilla.org/concepts/choosing_a_dataset_mobile.html#products-overview -SELECT - * REPLACE(REPLACE(usage, 'Firefox Non-desktop', app_name) AS usage) -FROM - joined diff --git a/templates/telemetry/smoot_usage_all_mtr_v1/query.sql b/templates/telemetry/smoot_usage_all_mtr_v1/query.sql deleted file mode 100644 index 1b1a1883bd3..00000000000 --- a/templates/telemetry/smoot_usage_all_mtr_v1/query.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT - * -FROM - smoot_usage_all_v1 diff --git a/templates/telemetry/smoot_usage_all_v1/view.sql b/templates/telemetry/smoot_usage_all_v1/view.sql deleted file mode 100644 index e0684160671..00000000000 --- a/templates/telemetry/smoot_usage_all_v1/view.sql +++ /dev/null @@ -1,21 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_all_v1` -AS -SELECT - * -FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_desktop_v1` --- -UNION ALL --- -SELECT - * -FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_nondesktop_v1` --- -UNION ALL --- -SELECT - * -FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_fxa_v1` diff --git a/templates/telemetry/smoot_usage_desktop_raw_v1/query.sql b/templates/telemetry/smoot_usage_desktop_raw_v1/query.sql deleted file mode 100644 index 386582bd267..00000000000 --- a/templates/telemetry/smoot_usage_desktop_raw_v1/query.sql +++ /dev/null @@ -1,101 +0,0 @@ --- Query generated by: templates/smoot_usage_raw.sql.py --source clients_last_seen_v1 - -WITH - base AS ( - SELECT - * REPLACE(normalized_channel AS channel) - FROM - clients_last_seen_v1) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ -- - STRUCT('Any Firefox Desktop Activity' AS usage, - STRUCT( - COUNTIF(days_since_seen < 1) AS dau, - COUNTIF(days_since_seen < 7) AS wau, - COUNTIF(days_since_seen < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_seen_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_seen_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 1) AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('Firefox Desktop Visited 5 URI' AS usage, - STRUCT( - COUNTIF(days_since_visited_5_uri < 1) AS dau, - COUNTIF(days_since_visited_5_uri < 7) AS wau, - COUNTIF(days_since_visited_5_uri < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_visited_5_uri_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_visited_5_uri_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 1) AND udf_active_n_weeks_ago(days_visited_5_uri_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('Firefox Desktop Opened Dev Tools' AS usage, - STRUCT( - COUNTIF(days_since_opened_dev_tools < 1) AS dau, - COUNTIF(days_since_opened_dev_tools < 7) AS wau, - COUNTIF(days_since_opened_dev_tools < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_opened_dev_tools_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_opened_dev_tools_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 1) AND udf_active_n_weeks_ago(days_opened_dev_tools_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('New Firefox Desktop Profile Created' AS usage, - STRUCT( - COUNTIF(days_since_created_profile < 1) AS dau, - NULL AS wau, - NULL AS mau, - NULL AS active_days_in_week - ) AS metrics_daily, - STRUCT( - NULL AS active_in_week_0, - NULL AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - NULL AS active_in_week_1, - NULL AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile) - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel diff --git a/templates/telemetry/smoot_usage_desktop_v1/view.sql b/templates/telemetry/smoot_usage_desktop_v1/view.sql deleted file mode 100644 index 62285763008..00000000000 --- a/templates/telemetry/smoot_usage_desktop_v1/view.sql +++ /dev/null @@ -1,76 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_desktop_v1` AS -WITH - base AS ( - SELECT - * - FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_desktop_raw_v1`), - -- - daily AS ( - SELECT - submission_date AS `date`, - metrics_daily.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week1 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 6 day) AS `date`, - metrics_1_week_post_new_profile.*, - * - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week2 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 13 day) AS `date`, - metrics_2_week_post_new_profile.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - joined AS ( - SELECT - * EXCEPT (submission_date, - metrics, - metrics_daily, - metrics_1_week_post_new_profile, - metrics_2_week_post_new_profile) - FROM - daily - FULL JOIN - new_profile_week1 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) - FULL JOIN - new_profile_week2 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) ) - -- -SELECT - * -FROM - joined diff --git a/templates/telemetry/smoot_usage_fxa_raw_v1/query.sql b/templates/telemetry/smoot_usage_fxa_raw_v1/query.sql deleted file mode 100644 index 1b11f1505c9..00000000000 --- a/templates/telemetry/smoot_usage_fxa_raw_v1/query.sql +++ /dev/null @@ -1,82 +0,0 @@ --- Query generated by: templates/smoot_usage_raw.sql.py --source fxa_users_last_seen_v1 - -WITH - base AS ( - SELECT - * REPLACE(country_names.code AS country), - user_id AS client_id, - days_since_registered AS days_since_created_profile, - language AS locale, - os_name AS os, - CAST(NULL AS STRING) AS app_name, - CAST(NULL AS STRING) AS channel - FROM - fxa_users_last_seen_v1 fxa - LEFT JOIN - static.country_names_v1 country_names - ON - fxa.country = country_names.`name` - ) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ -- - STRUCT('Any Firefox Account Activity' AS usage, - STRUCT( - COUNTIF(days_since_seen < 1) AS dau, - COUNTIF(days_since_seen < 7) AS wau, - COUNTIF(days_since_seen < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_seen_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_seen_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 1) AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('New Firefox Account Registered' AS usage, - STRUCT( - COUNTIF(days_since_registered < 1) AS dau, - NULL AS wau, - NULL AS mau, - NULL AS active_days_in_week - ) AS metrics_daily, - STRUCT( - NULL AS active_in_week_0, - NULL AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - NULL AS active_in_week_1, - NULL AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile) - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel diff --git a/templates/telemetry/smoot_usage_fxa_v1/view.sql b/templates/telemetry/smoot_usage_fxa_v1/view.sql deleted file mode 100644 index c75a02e7f29..00000000000 --- a/templates/telemetry/smoot_usage_fxa_v1/view.sql +++ /dev/null @@ -1,76 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_fxa_v1` AS -WITH - base AS ( - SELECT - * - FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_fxa_raw_v1` ), - -- - daily AS ( - SELECT - submission_date AS `date`, - metrics_daily.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week1 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 6 day) AS `date`, - metrics_1_week_post_new_profile.*, - * - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week2 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 13 day) AS `date`, - metrics_2_week_post_new_profile.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - joined AS ( - SELECT - * EXCEPT (submission_date, - metrics, - metrics_daily, - metrics_1_week_post_new_profile, - metrics_2_week_post_new_profile) - FROM - daily - FULL JOIN - new_profile_week1 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) - FULL JOIN - new_profile_week2 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) ) - -- -SELECT - * -FROM - joined diff --git a/templates/telemetry/smoot_usage_nondesktop_raw_v1/query.sql b/templates/telemetry/smoot_usage_nondesktop_raw_v1/query.sql deleted file mode 100644 index 67640134eef..00000000000 --- a/templates/telemetry/smoot_usage_nondesktop_raw_v1/query.sql +++ /dev/null @@ -1,86 +0,0 @@ --- Query generated by: templates/smoot_usage_raw.sql.py --source nondesktop_clients_last_seen_v1 - -WITH - base AS ( - SELECT - *, - normalized_channel AS channel - FROM - nondesktop_clients_last_seen_v1 - WHERE - -- We apply this filter here rather than in the live view because this field - -- is not normalized and there are many single pings that come in with unique - -- nonsensical app_name values. App names are documented in - -- https://docs.telemetry.mozilla.org/concepts/choosing_a_dataset_mobile.html#products-overview - (STARTS_WITH(app_name, 'FirefoxReality') OR app_name IN ( - 'Fenix', - 'Fennec', -- Firefox for Android and Firefox for iOS - 'Focus', - 'FirefoxConnect', -- Amazon Echo - 'FirefoxForFireTV', - 'Zerda')) -- Firefox Lite, previously called Rocket - -- There are also many strange nonsensical entries for os, so we filter here. - AND os IN ('Android', 'iOS')) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ -- - STRUCT('Any Firefox Non-desktop Activity' AS usage, - STRUCT( - COUNTIF(days_since_seen < 1) AS dau, - COUNTIF(days_since_seen < 7) AS wau, - COUNTIF(days_since_seen < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_seen_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_seen_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_seen_bits, 1) AND udf_active_n_weeks_ago(days_seen_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile), - STRUCT('New Firefox Non-desktop Profile Created' AS usage, - STRUCT( - COUNTIF(days_since_created_profile < 1) AS dau, - NULL AS wau, - NULL AS mau, - NULL AS active_days_in_week - ) AS metrics_daily, - STRUCT( - NULL AS active_in_week_0, - NULL AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - NULL AS active_in_week_1, - NULL AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile) - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel diff --git a/templates/telemetry/smoot_usage_nondesktop_v1/view.sql b/templates/telemetry/smoot_usage_nondesktop_v1/view.sql deleted file mode 100644 index b3e4c9204ef..00000000000 --- a/templates/telemetry/smoot_usage_nondesktop_v1/view.sql +++ /dev/null @@ -1,91 +0,0 @@ -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.smoot_usage_nondesktop_v1` AS -WITH - base AS ( - SELECT - * - FROM - `moz-fx-data-derived-datasets.telemetry.smoot_usage_nondesktop_raw_v1` ), - -- - daily AS ( - SELECT - submission_date AS `date`, - metrics_daily.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week1 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 6 day) AS `date`, - metrics_1_week_post_new_profile.*, - * - FROM - base, - UNNEST(metrics) ), - -- - new_profile_week2 AS ( - SELECT - DATE_SUB(submission_date, INTERVAL 13 day) AS `date`, - metrics_2_week_post_new_profile.*, - * EXCEPT (new_profiles) - FROM - base, - UNNEST(metrics) ), - -- - joined AS ( - SELECT - * EXCEPT (submission_date, - metrics, - metrics_daily, - metrics_1_week_post_new_profile, - metrics_2_week_post_new_profile) - REPLACE ( - CASE app_name - WHEN 'Fennec' THEN CONCAT(app_name, ' ', os) - WHEN 'Focus' THEN CONCAT(app_name, ' ', os) - WHEN 'Zerda' THEN 'Firefox Lite' - ELSE app_name - END AS app_name) - FROM - daily - FULL JOIN - new_profile_week1 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) - FULL JOIN - new_profile_week2 - USING - (`date`, - usage, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel) ) - -- -SELECT - * -FROM - joined - -- -UNION ALL - -- Also present each app as its own usage criterion. App names are documented in - -- https://docs.telemetry.mozilla.org/concepts/choosing_a_dataset_mobile.html#products-overview -SELECT - * REPLACE(REPLACE(usage, 'Firefox Non-desktop', app_name) AS usage) -FROM - joined diff --git a/templates/telemetry/smoot_usage_raw.sql.py b/templates/telemetry/smoot_usage_raw.sql.py deleted file mode 100755 index d3a29e44633..00000000000 --- a/templates/telemetry/smoot_usage_raw.sql.py +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env python3 -""" -Smoot metrics query generator. - -The --source parameter determines where this operates on desktop -or nondesktop data. The output table contains a metrics array -where each entry correponds to a particular usage criterion and -contains a series of structs representing different points in time. - -The "daily" metrics include all profiles from the given day in the -source table and may represent some history of the preceeding days -per the methodology of clients_last_seen where we carry over the -most recent observation for 28 days. - -The "1 week post-profile" and "2-week post-profile" structs give us a -way to express metrics that depend on activity in a forward-looking -window rather than a backwards-looking window. The 1 week metrics -consider only profiles that are exactly 6 days old; this is the day on -which we can calculate metrics like how many profiles satisfied a given -usage criterion in their first week. But we generally want to associate -these metrics with the day the profile was created, so we need to shift -the metric back by a week when presenting to the user. That responsbility -is handled by the views (such as smoot_desktop_usage) on top of these raw -tables. - -The user-facing views will show null values for all 1-week metrics until -the target day is at least 7 days old. For example, 2019-01-01 will show -null values for 1-week metrics if viewed on 2019-01-06, but once 2019-01-07 -values are processed, the user-facing view will have the 1-week metrics -populated. -""" - -import argparse -from dataclasses import dataclass, asdict -import sys -from textwrap import dedent, indent - - -parser = argparse.ArgumentParser() -parser.add_argument("--source", type=str, help="source table to query", required=True) - - -TEMPLATE = """\ --- Query generated by: templates/smoot_usage_raw.sql.py --source {source} - -WITH - base AS ( -{base_select}) - -- -SELECT - submission_date, - COUNTIF(days_since_created_profile = 6) AS new_profiles, - [ --{usage_structs} - ] AS metrics, - -- We hash client_ids into 20 buckets to aid in computing - -- confidence intervals for mau/wau/dau sums; the particular hash - -- function and number of buckets is subject to change in the future. - MOD(ABS(FARM_FINGERPRINT(client_id)), 20) AS id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel -FROM - base -WHERE - client_id IS NOT NULL - -- Reprocess all dates by running this query with --parameter=submission_date:DATE:NULL - AND (@submission_date IS NULL OR @submission_date = submission_date) -GROUP BY - submission_date, - id_bucket, - app_name, - app_version, - country, - locale, - os, - os_version, - channel""" # noqa - - -@dataclass -class UsageCriterion: - """Wraps a logical criterion name with a column name and generates SQL.""" - - display_name: str - col_name: str - dau_only: bool = False - - @property - def sql(self): - """Return a SQL string representing metrics on this criterion.""" - text = """ - STRUCT('{display_name}' AS usage, - STRUCT( - COUNTIF(days_since_{col_name} < 1) AS dau, - COUNTIF(days_since_{col_name} < 7) AS wau, - COUNTIF(days_since_{col_name} < 28) AS mau, - SUM(udf_bitcount_lowest_7(days_{col_name}_bits)) AS active_days_in_week - ) AS metrics_daily, - STRUCT( - COUNTIF(days_since_created_profile = 6 AND udf_active_n_weeks_ago(days_{col_name}_bits, 0)) AS active_in_week_0, - SUM(IF(days_since_created_profile = 6, udf_bitcount_lowest_7(days_{col_name}_bits), 0)) AS active_days_in_week_0 - ) AS metrics_1_week_post_new_profile, - STRUCT( - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_{col_name}_bits, 0)) AS active_in_week_1, - COUNTIF(days_since_created_profile = 13 AND udf_active_n_weeks_ago(days_{col_name}_bits, 1) AND udf_active_n_weeks_ago(days_{col_name}_bits, 0)) AS active_in_weeks_0_and_1 - ) AS metrics_2_week_post_new_profile)""".format( # noqa - **asdict(self) - ) - if self.dau_only: - lines = [] - for line in text.split("\n"): - before_as, separator, after_as = line.partition(" AS ") - if separator and ( - after_as not in ["usage,", "dau,"] - and not after_as.startswith("metrics_") - ): - line = " NULL AS " + after_as - lines.append(line) - text = "\n".join(lines) - return text - - -USAGE_CRITERIA = { - "clients_last_seen_v1": ( - UsageCriterion("Any Firefox Desktop Activity", "seen"), - UsageCriterion("Firefox Desktop Visited 5 URI", "visited_5_uri"), - UsageCriterion("Firefox Desktop Opened Dev Tools", "opened_dev_tools"), - UsageCriterion( - "New Firefox Desktop Profile Created", "created_profile", dau_only=True - ), - ), - "nondesktop_clients_last_seen_v1": ( - UsageCriterion("Any Firefox Non-desktop Activity", "seen"), - UsageCriterion( - "New Firefox Non-desktop Profile Created", "created_profile", dau_only=True - ), - ), - "fxa_users_last_seen_v1": ( - UsageCriterion("Any Firefox Account Activity", "seen"), - UsageCriterion("New Firefox Account Registered", "registered", dau_only=True), - ), -} - - -BASE_SELECT = { - "clients_last_seen_v1": """\ - SELECT - * REPLACE(normalized_channel AS channel) - FROM - clients_last_seen_v1""", - "nondesktop_clients_last_seen_v1": """\ - SELECT - *, - normalized_channel AS channel - FROM - nondesktop_clients_last_seen_v1 - WHERE - -- We apply this filter here rather than in the live view because this field - -- is not normalized and there are many single pings that come in with unique - -- nonsensical app_name values. App names are documented in - -- https://docs.telemetry.mozilla.org/concepts/choosing_a_dataset_mobile.html#products-overview - (STARTS_WITH(app_name, 'FirefoxReality') OR app_name IN ( - 'Fenix', - 'Fennec', -- Firefox for Android and Firefox for iOS - 'Focus', - 'FirefoxConnect', -- Amazon Echo - 'FirefoxForFireTV', - 'Zerda')) -- Firefox Lite, previously called Rocket - -- There are also many strange nonsensical entries for os, so we filter here. - AND os IN ('Android', 'iOS')""", # noqa - "fxa_users_last_seen_v1": """\ - SELECT - *, - user_id AS client_id, - days_since_registered AS days_since_created_profile, - language AS locale, - os_name AS os, - NULL AS app_name, - NULL AS channel - FROM - fxa_users_last_seen_v1""", -} - - -def generate_sql(opts): - """Return the text of a full SQL query based on the given opts.""" - base_select = BASE_SELECT[opts["source"]] - usage_structs = ",".join(u.sql for u in USAGE_CRITERIA[opts["source"]]) - usage_structs = indent(dedent(usage_structs), " ") - return TEMPLATE.format(**locals(), **opts) - - -def main(argv, out=print): - """Print a smoot query to stdout.""" - opts = parser.parse_args(argv[1:]) - out(generate_sql(vars(opts))) - - -if __name__ == "__main__": - main(sys.argv)