From c067b8b93f02b7467b1dad46925d159f7b1aa619 Mon Sep 17 00:00:00 2001 From: Arkadiusz Komarzewski Date: Fri, 24 Nov 2023 15:43:37 +0100 Subject: [PATCH 1/2] Bug 1866469 - Exclude use_counters from GLAM ETL --- script/glam/generate_glean_sql | 3 +++ 1 file changed, 3 insertions(+) diff --git a/script/glam/generate_glean_sql b/script/glam/generate_glean_sql index 4b971f4da8e..60f6dd88e48 100755 --- a/script/glam/generate_glean_sql +++ b/script/glam/generate_glean_sql @@ -60,9 +60,12 @@ function write_clients_daily_aggregates { # e.g. baseline_v1 local tables; + # we're excluding use_counters because they are not supported in GLAM + # and generated query is too big to run tables=$( bq ls "$qualified" \ | grep TABLE \ + | grep -v use_counters \ | awk '{print $1}' ) # generate all of the schemas in parallel From bfdf7bf00a34e55baf193e57033601b80e0e5191 Mon Sep 17 00:00:00 2001 From: Eduardo Filho Date: Fri, 24 Nov 2023 12:51:27 -0500 Subject: [PATCH 2/2] Attempt to fix tests #1 --- .../expect.yaml | 14 +++++++------- .../test_first_session_no_baseline/expect.yaml | 6 +++--- .../test_first_session_seq/expect.yaml | 4 ++-- .../test_only_metrics_ping/expect.yaml | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_baseline_no_first_session_with_activations/expect.yaml b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_baseline_no_first_session_with_activations/expect.yaml index 23ae939601b..c9c8d7bdd22 100644 --- a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_baseline_no_first_session_with_activations/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_baseline_no_first_session_with_activations/expect.yaml @@ -9,31 +9,31 @@ last_reported_device_model: model-2 last_reported_channel: release activated: true - install_source: com.android.vending channel: release + install_source: com.android.vending metadata: + install_source__source_ping: metrics + install_source__source_ping_datetime: 2023-01-01T00:00:00 + min_metrics_ping_submission_date: 2023-01-01 reported_metrics_ping: true reported_first_session_ping: false reported_baseline_ping: true - min_metrics_ping_submission_date: 2023-01-01 - install_source__source_ping_datetime: '2023-01-01T00:00:00' - install_source__source_ping: metrics - client_id: client-2 sample_id: 50 first_seen_date: 2022-12-25 submission_date: 2022-12-25 device_model: model-2 + last_reported_device_model: model-2 adjust_network: Organic last_reported_adjust_network: Organic - last_reported_device_model: model-2 activated: false metadata: + adjust_network__source_ping: metrics + adjust_network__source_ping_datetime: 2023-01-01T00:00:00 reported_metrics_ping: true reported_first_session_ping: false reported_baseline_ping: false min_metrics_ping_submission_date: 2023-01-01 - adjust_network__source_ping: metrics - adjust_network__source_ping_datetime: '2023-01-01T00:00:00' - client_id: client-3 sample_id: 50 adjust_network: Organic diff --git a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_no_baseline/expect.yaml b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_no_baseline/expect.yaml index 602515f07c6..75617d23d82 100644 --- a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_no_baseline/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_no_baseline/expect.yaml @@ -30,6 +30,7 @@ adjust_network__source_ping_datetime: '2023-05-02T00:00:00' - client_id: client-3 sample_id: 70 + install_source: com.android.vending first_seen_date: 2023-05-01 submission_date: 2023-05-02 first_run_date: 2023-05-01 @@ -37,11 +38,10 @@ last_reported_channel: release channel: release activated: false - install_source: com.android.vending metadata: + install_source__source_ping_datetime: 2023-05-02T00:00:00 + min_metrics_ping_submission_date: 2023-05-02 reported_first_session_ping: false reported_metrics_ping: true reported_baseline_ping: true install_source__source_ping: metrics - install_source__source_ping_datetime: '2023-05-02T00:00:00' - min_metrics_ping_submission_date: 2023-05-02 diff --git a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_seq/expect.yaml b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_seq/expect.yaml index c7435bee6d9..5785f15f29a 100644 --- a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_seq/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_first_session_seq/expect.yaml @@ -31,6 +31,7 @@ - client_id: client-3 sample_id: 70 first_seen_date: 2023-05-01 + install_source: com.android.vending submission_date: 2023-05-02 first_run_date: 2023-05-01 last_reported_date: 2023-05-02 @@ -38,13 +39,12 @@ adjust_network: Untrusted Devices channel: release activated: false - install_source: com.android.vending metadata: reported_first_session_ping: true reported_metrics_ping: true reported_baseline_ping: true install_source__source_ping: metrics - install_source__source_ping_datetime: '2023-05-02T00:00:00' + install_source__source_ping_datetime: 2023-05-02T00:00:00 min_metrics_ping_submission_date: 2023-05-02 min_first_session_ping_submission_date: 2023-05-02 min_first_session_ping_run_date: 2023-05-01 diff --git a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_only_metrics_ping/expect.yaml b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_only_metrics_ping/expect.yaml index c307a1de2b7..049a895ba19 100644 --- a/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_only_metrics_ping/expect.yaml +++ b/tests/sql/moz-fx-data-shared-prod/fenix_derived/firefox_android_clients_v1/test_only_metrics_ping/expect.yaml @@ -9,16 +9,16 @@ last_reported_country: CA sample_id: 54 activated: false - adjust_network: 'Organic' - last_reported_adjust_network: 'Organic' + adjust_network: Organic + last_reported_adjust_network: Organic channel: release metadata: + adjust_network__source_ping: metrics + adjust_network__source_ping_datetime: 2023-01-01T00:00:00 + min_metrics_ping_submission_date: 2023-01-01 reported_first_session_ping: false reported_metrics_ping: true reported_baseline_ping: true - min_metrics_ping_submission_date: 2023-01-01 - adjust_network__source_ping_datetime: '2023-01-01T00:00:00' - adjust_network__source_ping: metrics - client_id: client-2 install_source: 'Test Install Source' sample_id: 40