From d2dd84019c0422c1efc60d501294c72053daeee2 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Thu, 30 Mar 2023 19:17:24 -0700 Subject: [PATCH] fix(dbt): dbt Cloud columns as the standard --- src/preset_cli/api/clients/dbt.py | 4 ++++ src/preset_cli/cli/superset/sync/dbt/command.py | 1 + src/preset_cli/cli/superset/sync/dbt/datasets.py | 7 ++++--- tests/cli/superset/sync/dbt/command_test.py | 4 ++-- tests/cli/superset/sync/dbt/datasets_test.py | 14 +++++++------- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/preset_cli/api/clients/dbt.py b/src/preset_cli/api/clients/dbt.py index 131a4c5b..b419c2e8 100644 --- a/src/preset_cli/api/clients/dbt.py +++ b/src/preset_cli/api/clients/dbt.py @@ -673,6 +673,10 @@ def get_models(self, job_id: int) -> List[ModelSchema]: description meta tags + columns { + name + description + } } } """ diff --git a/src/preset_cli/cli/superset/sync/dbt/command.py b/src/preset_cli/cli/superset/sync/dbt/command.py index 83317b24..e8115792 100644 --- a/src/preset_cli/cli/superset/sync/dbt/command.py +++ b/src/preset_cli/cli/superset/sync/dbt/command.py @@ -130,6 +130,7 @@ def dbt_core( # pylint: disable=too-many-arguments, too-many-locals # conform to the same schema that dbt Cloud uses for models unique_id = config["uniqueId"] = config["unique_id"] config["children"] = configs["child_map"][unique_id] + config["columns"] = list(config["columns"].values()) models.append(model_schema.load(config)) models = apply_select(models, select, exclude) model_map = { diff --git a/src/preset_cli/cli/superset/sync/dbt/datasets.py b/src/preset_cli/cli/superset/sync/dbt/datasets.py index 05c2b0a0..b24e4a48 100644 --- a/src/preset_cli/cli/superset/sync/dbt/datasets.py +++ b/src/preset_cli/cli/superset/sync/dbt/datasets.py @@ -154,12 +154,13 @@ def sync_datasets( # pylint: disable=too-many-locals, too-many-branches, too-ma # update column descriptions if columns := model.get("columns"): + column_metadata = {column["name"]: column for column in columns} current_columns = client.get_dataset(dataset["id"])["columns"] for column in current_columns: name = column["column_name"] - if name in columns: - column["description"] = columns[name].get("description", "") - column["label"] = columns[name].get("label", "") + if name in column_metadata: + column["description"] = column_metadata[name].get("description", "") + column["verbose_name"] = column_metadata[name].get("name", "") # remove columns that are not part of the update payload for key in ("changed_on", "created_on", "type_generic"): diff --git a/tests/cli/superset/sync/dbt/command_test.py b/tests/cli/superset/sync/dbt/command_test.py index 4b63d582..aa394872 100644 --- a/tests/cli/superset/sync/dbt/command_test.py +++ b/tests/cli/superset/sync/dbt/command_test.py @@ -82,7 +82,7 @@ def test_dbt_core(mocker: MockerFixture, fs: FakeFilesystem) -> None: models = [ { "database": "examples_dev", - "columns": {}, + "columns": [], "meta": {}, "description": "", "name": "messages_channels", @@ -341,7 +341,7 @@ def test_dbt(mocker: MockerFixture, fs: FakeFilesystem) -> None: { "meta": {}, "tags": [], - "columns": {}, + "columns": [], "schema": "public", "name": "messages_channels", "database": "examples_dev", diff --git a/tests/cli/superset/sync/dbt/datasets_test.py b/tests/cli/superset/sync/dbt/datasets_test.py index a883a93c..c7c7054c 100644 --- a/tests/cli/superset/sync/dbt/datasets_test.py +++ b/tests/cli/superset/sync/dbt/datasets_test.py @@ -45,7 +45,7 @@ "meta": {}, "name": "messages_channels", "unique_id": "model.superset_examples.messages_channels", - "columns": {"id": {"description": "Primary key", "label": "some label"}}, + "columns": [{"name": "id", "description": "Primary key"}], }, ), ] @@ -115,8 +115,8 @@ def test_sync_datasets_new(mocker: MockerFixture) -> None: { "column_name": "id", "description": "Primary key", - "label": "some label", "is_dttm": False, + "verbose_name": "id", }, { "column_name": "ts", @@ -152,7 +152,7 @@ def test_sync_datasets_with_alias(mocker: MockerFixture) -> None: "meta": {}, "name": "messages_channels", "unique_id": "model.superset_examples.messages_channels", - "columns": {"id": {"description": "Primary key"}}, + "columns": [{"name": "id", "description": "Primary key"}], }, ), ] @@ -206,8 +206,8 @@ def test_sync_datasets_with_alias(mocker: MockerFixture) -> None: { "column_name": "id", "description": "Primary key", - "label": "", "is_dttm": False, + "verbose_name": "id", }, { "column_name": "ts", @@ -266,8 +266,8 @@ def test_sync_datasets_no_metrics(mocker: MockerFixture) -> None: { "column_name": "id", "description": "Primary key", - "label": "some label", "is_dttm": False, + "verbose_name": "id", }, ], ), @@ -359,8 +359,8 @@ def test_sync_datasets_existing(mocker: MockerFixture) -> None: { "column_name": "id", "description": "Primary key", - "label": "some label", "is_dttm": False, + "verbose_name": "id", }, ], ), @@ -447,8 +447,8 @@ def test_sync_datasets_external_url(mocker: MockerFixture) -> None: { "column_name": "id", "description": "Primary key", - "label": "some label", "is_dttm": False, + "verbose_name": "id", }, ], ),