Merge pull request #197 from preset-io/sc-68582

fix(dbt): dbt Cloud columns as the standard
preset-io · Mar 31, 2023 · c5d9f99 · c5d9f99
2 parents bc2d996 + d2dd840
commit c5d9f99
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 12 deletions.
diff --git a/src/preset_cli/api/clients/dbt.py b/src/preset_cli/api/clients/dbt.py
@@ -673,6 +673,10 @@ def get_models(self, job_id: int) -> List[ModelSchema]:
                     description
                     meta
                     tags
+                    columns {
+                        name
+                        description
+                    }
                 }
             }
         """

diff --git a/src/preset_cli/cli/superset/sync/dbt/command.py b/src/preset_cli/cli/superset/sync/dbt/command.py
@@ -130,6 +130,7 @@ def dbt_core(  # pylint: disable=too-many-arguments, too-many-locals
             # conform to the same schema that dbt Cloud uses for models
             unique_id = config["uniqueId"] = config["unique_id"]
             config["children"] = configs["child_map"][unique_id]
+            config["columns"] = list(config["columns"].values())
             models.append(model_schema.load(config))
     models = apply_select(models, select, exclude)
     model_map = {

diff --git a/src/preset_cli/cli/superset/sync/dbt/datasets.py b/src/preset_cli/cli/superset/sync/dbt/datasets.py
@@ -154,12 +154,13 @@ def sync_datasets(  # pylint: disable=too-many-locals, too-many-branches, too-ma
 
         # update column descriptions
         if columns := model.get("columns"):
+            column_metadata = {column["name"]: column for column in columns}
             current_columns = client.get_dataset(dataset["id"])["columns"]
             for column in current_columns:
                 name = column["column_name"]
-                if name in columns:
-                    column["description"] = columns[name].get("description", "")
-                    column["label"] = columns[name].get("label", "")
+                if name in column_metadata:
+                    column["description"] = column_metadata[name].get("description", "")
+                    column["verbose_name"] = column_metadata[name].get("name", "")
 
                 # remove columns that are not part of the update payload
                 for key in ("changed_on", "created_on", "type_generic"):

diff --git a/tests/cli/superset/sync/dbt/command_test.py b/tests/cli/superset/sync/dbt/command_test.py
@@ -82,7 +82,7 @@ def test_dbt_core(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     models = [
         {
             "database": "examples_dev",
-            "columns": {},
+            "columns": [],
             "meta": {},
             "description": "",
             "name": "messages_channels",
@@ -341,7 +341,7 @@ def test_dbt(mocker: MockerFixture, fs: FakeFilesystem) -> None:
         {
             "meta": {},
             "tags": [],
-            "columns": {},
+            "columns": [],
             "schema": "public",
             "name": "messages_channels",
             "database": "examples_dev",

diff --git a/tests/cli/superset/sync/dbt/datasets_test.py b/tests/cli/superset/sync/dbt/datasets_test.py
@@ -45,7 +45,7 @@
             "meta": {},
             "name": "messages_channels",
             "unique_id": "model.superset_examples.messages_channels",
-            "columns": {"id": {"description": "Primary key", "label": "some label"}},
+            "columns": [{"name": "id", "description": "Primary key"}],
         },
     ),
 ]
@@ -115,8 +115,8 @@ def test_sync_datasets_new(mocker: MockerFixture) -> None:
                     {
                         "column_name": "id",
                         "description": "Primary key",
-                        "label": "some label",
                         "is_dttm": False,
+                        "verbose_name": "id",
                     },
                     {
                         "column_name": "ts",
@@ -152,7 +152,7 @@ def test_sync_datasets_with_alias(mocker: MockerFixture) -> None:
                 "meta": {},
                 "name": "messages_channels",
                 "unique_id": "model.superset_examples.messages_channels",
-                "columns": {"id": {"description": "Primary key"}},
+                "columns": [{"name": "id", "description": "Primary key"}],
             },
         ),
     ]
@@ -206,8 +206,8 @@ def test_sync_datasets_with_alias(mocker: MockerFixture) -> None:
                     {
                         "column_name": "id",
                         "description": "Primary key",
-                        "label": "",
                         "is_dttm": False,
+                        "verbose_name": "id",
                     },
                     {
                         "column_name": "ts",
@@ -266,8 +266,8 @@ def test_sync_datasets_no_metrics(mocker: MockerFixture) -> None:
                     {
                         "column_name": "id",
                         "description": "Primary key",
-                        "label": "some label",
                         "is_dttm": False,
+                        "verbose_name": "id",
                     },
                 ],
             ),
@@ -359,8 +359,8 @@ def test_sync_datasets_existing(mocker: MockerFixture) -> None:
                     {
                         "column_name": "id",
                         "description": "Primary key",
-                        "label": "some label",
                         "is_dttm": False,
+                        "verbose_name": "id",
                     },
                 ],
             ),
@@ -447,8 +447,8 @@ def test_sync_datasets_external_url(mocker: MockerFixture) -> None:
                     {
                         "column_name": "id",
                         "description": "Primary key",
-                        "label": "some label",
                         "is_dttm": False,
+                        "verbose_name": "id",
                     },
                 ],
             ),