feat (dbt): pass dbt_project.yml

src/preset_cli/cli/superset/sync/dbt/command.py

@@ -3,6 +3,8 @@
 """
 
 import os.path
+import sys
+import warnings
 from pathlib import Path
 from typing import Optional, Tuple
 
@@ -22,8 +24,8 @@
 
 
 @click.command()
-@click.argument("manifest", type=click.Path(exists=True, resolve_path=True))
-@click.option("--project", help="Name of the dbt project", default="default")
+@click.argument("file", type=click.Path(exists=True, resolve_path=True))
+@click.option("--project", help="Name of the dbt project", default=None)
 @click.option("--target", help="Target name", default=None)
 @click.option(
     "--profiles",
@@ -63,8 +65,8 @@
 @click.pass_context
 def dbt_core(  # pylint: disable=too-many-arguments, too-many-locals
     ctx: click.core.Context,
-    manifest: str,
-    project: str,
+    file: str,
+    project: Optional[str],
     target: Optional[str],
     select: Tuple[str, ...],
     exclude: Tuple[str, ...],
@@ -84,11 +86,40 @@ def dbt_core(  # pylint: disable=too-many-arguments, too-many-locals
     if profiles is None:
         profiles = os.path.expanduser("~/.dbt/profiles.yml")
 
+    file_path = Path(file)
+    if file_path.name == "manifest.json":
+        warnings.warn(
+            (
+                "Passing the manifest.json file is deprecated. "
+                "Please pass the dbt_project.yml file instead."
+            ),
+            category=DeprecationWarning,
+            stacklevel=2,
+        )
+        manifest = file_path
+        profile = project = project or "default"
+    elif file_path.name == "dbt_project.yml":
+        with open(file_path, encoding="utf-8") as input_:
+            dbt_project = yaml.load(input_, Loader=yaml.SafeLoader)
+
+        manifest = file_path.parent / dbt_project["target-path"] / "manifest.json"
+        profile = dbt_project["profile"]
+        project = project or dbt_project["name"]
+    else:
+        click.echo(
+            click.style(
+                "FILE should be either manifest.json or dbt_project.yml",
+                fg="bright_red",
+            ),
+        )
+        sys.exit(1)
+
     try:
         database = sync_database(
             client,
             Path(profiles),
             project,
+            profile,
             target,
             import_db,
             disallow_edits,

src/preset_cli/cli/superset/sync/dbt/databases.py

@@ -19,6 +19,7 @@ def sync_database(  # pylint: disable=too-many-locals, too-many-arguments
     client: SupersetClient,
     profiles_path: Path,
     project_name: str,
+    profile_name: str,
     target_name: Optional[str],
     import_db: bool,
     disallow_edits: bool,  # pylint: disable=unused-argument
@@ -29,7 +30,7 @@ def sync_database(  # pylint: disable=too-many-locals, too-many-arguments
     """
     base_url = URL(external_url_prefix) if external_url_prefix else None
 
-    profiles = load_profiles(profiles_path, project_name, target_name)
+    profiles = load_profiles(profiles_path, project_name, profile_name, target_name)
     project = profiles[project_name]
     outputs = project["outputs"]
     if target_name is None:

src/preset_cli/cli/superset/sync/dbt/lib.py

@@ -232,17 +232,20 @@ class Target(TypedDict):
 
 
 def load_profiles(
-    path: Path, project_name: str, target_name: Optional[str],
+    path: Path,
+    project_name: str,
+    profile_name: str,
+    target_name: Optional[str],
 ) -> Dict[str, Any]:
     """
     Load the file and apply Jinja2 templating.
     """
     with open(path, encoding="utf-8") as input_:
         profiles = yaml.load(input_, Loader=yaml.SafeLoader)
 
-    if project_name not in profiles:
-        raise Exception(f"Project {project_name} not found in {path}")
-    project = profiles[project_name]
+    if profile_name not in profiles:
+        raise Exception(f"Project {profile_name} not found in {path}")
+    project = profiles[profile_name]
     outputs = project["outputs"]
     if target_name is None:
         target_name = project["target"]
@@ -259,6 +262,7 @@ def load_profiles(
     context = {
         "env_var": env_var,
         "project_name": project_name,
+        "profile_name": profile_name,
         "target": target,
     }
 

tests/cli/superset/sync/dbt/command_test.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 
 import pytest
+import yaml
 from click.testing import CliRunner
 from pyfakefs.fake_filesystem import FakeFilesystem
 from pytest_mock import MockerFixture
@@ -67,6 +68,7 @@ def test_dbt_core(mocker: MockerFixture, fs: FakeFilesystem) -> None:
         client,
         profiles,
         "default",
+        "default",
         None,
         False,
         False,
@@ -107,6 +109,92 @@ def test_dbt_core(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     sync_exposures.assert_called_with(client, exposures, sync_datasets())
 
 
+def test_dbt_core_dbt_project(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """
+    Test the ``dbt-core`` command with a ``dbt_project.yml`` file.
+    """
+    root = Path("/path/to/root")
+    fs.create_dir(root)
+    dbt_project = root / "default/dbt_project.yml"
+    fs.create_file(
+        dbt_project,
+        contents=yaml.dump(
+            {
+                "name": "my_project",
+                "profile": "default",
+                "target-path": "target",
+            },
+        ),
+    )
+    manifest = root / "default/target/manifest.json"
+    fs.create_file(manifest, contents=manifest_contents)
+    profiles = root / ".dbt/profiles.yml"
+    fs.create_file(profiles)
+
+    SupersetClient = mocker.patch(
+        "preset_cli.cli.superset.sync.dbt.command.SupersetClient",
+    )
+    client = SupersetClient()
+    mocker.patch("preset_cli.cli.superset.main.UsernamePasswordAuth")
+    sync_database = mocker.patch(
+        "preset_cli.cli.superset.sync.dbt.command.sync_database",
+    )
+
+    runner = CliRunner()
+    result = runner.invoke(
+        superset_cli,
+        [
+            "https://superset.example.org/",
+            "sync",
+            "dbt-core",
+            str(dbt_project),
+            "--profiles",
+            str(profiles),
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 0
+    sync_database.assert_called_with(
+        client,
+        profiles,
+        "my_project",
+        "default",
+        None,
+        False,
+        False,
+        "",
+    )
+
+
+def test_dbt_core_invalid_argument(mocker: MockerFixture, fs: FakeFilesystem) -> None:
+    """
+    Test the ``dbt-core`` command with an invalid argument.
+    """
+    root = Path("/path/to/root")
+    fs.create_dir(root)
+    profiles = root / ".dbt/profiles.yml"
+    fs.create_file(profiles)
+    wrong = root / "wrong"
+    fs.create_file(wrong)
+
+    mocker.patch("preset_cli.cli.superset.sync.dbt.command.SupersetClient")
+    mocker.patch("preset_cli.cli.superset.main.UsernamePasswordAuth")
+
+    runner = CliRunner()
+    result = runner.invoke(
+        superset_cli,
+        [
+            "https://superset.example.org/",
+            "sync",
+            "dbt-core",
+            str(wrong),
+        ],
+        catch_exceptions=False,
+    )
+    assert result.exit_code == 1
+    assert result.output == "FILE should be either manifest.json or dbt_project.yml\n"
+
+
 def test_dbt(mocker: MockerFixture, fs: FakeFilesystem) -> None:
     """
     Test the ``dbt`` command.
@@ -158,6 +246,7 @@ def test_dbt(mocker: MockerFixture, fs: FakeFilesystem) -> None:
         client,
         profiles,
         "default",
+        "default",
         None,
         False,
         False,
@@ -279,6 +368,7 @@ def test_dbt_core_default_profile(mocker: MockerFixture, fs: FakeFilesystem) ->
         client,
         profiles,
         "default",
+        "default",
         None,
         False,
         False,

tests/cli/superset/sync/dbt/databases_test.py

@@ -33,6 +33,7 @@ def test_sync_database_new(mocker: MockerFixture, fs: FakeFilesystem) -> None:
         client=client,
         profiles_path=Path("/path/to/.dbt/profiles.yml"),
         project_name="my_project",
+        profile_name="my_project",
         target_name="dev",
         import_db=True,
         disallow_edits=False,
@@ -48,7 +49,8 @@ def test_sync_database_new(mocker: MockerFixture, fs: FakeFilesystem) -> None:
 
 
 def test_sync_database_new_default_target(
-    mocker: MockerFixture, fs: FakeFilesystem
+    mocker: MockerFixture,
+    fs: FakeFilesystem,
 ) -> None:
     """
     Test ``sync_database`` when we want to import a new DB using the default target.
@@ -68,6 +70,7 @@ def test_sync_database_new_default_target(
         client=client,
         profiles_path=Path("/path/to/.dbt/profiles.yml"),
         project_name="my_project",
+        profile_name="my_project",
         target_name=None,
         import_db=True,
         disallow_edits=False,
@@ -121,6 +124,7 @@ def test_sync_database_new_custom_sqlalchemy_uri(
         client=client,
         profiles_path=Path("/path/to/.dbt/profiles.yml"),
         project_name="my_project",
+        profile_name="my_project",
         target_name="dev",
         import_db=True,
         disallow_edits=False,
@@ -177,6 +181,7 @@ def test_sync_database_env_var(
         client=client,
         profiles_path=Path("/path/to/.dbt/profiles.yml"),
         project_name="my_project",
+        profile_name="my_project",
         target_name="dev",
         import_db=True,
         disallow_edits=False,
@@ -206,7 +211,8 @@ def test_sync_database_no_project(mocker: MockerFixture, fs: FakeFilesystem) ->
         sync_database(
             client=client,
             profiles_path=Path("/path/to/.dbt/profiles.yml"),
-            project_name="my_other_project",
+            project_name="my_project",
+            profile_name="my_other_project",
             target_name="dev",
             import_db=True,
             disallow_edits=False,
@@ -234,6 +240,7 @@ def test_sync_database_no_target(mocker: MockerFixture, fs: FakeFilesystem) -> N
             client=client,
             profiles_path=Path("/path/to/.dbt/profiles.yml"),
             project_name="my_project",
+            profile_name="my_project",
             target_name="prod",
             import_db=True,
             disallow_edits=False,
@@ -273,6 +280,7 @@ def test_sync_database_multiple_databases(
             client=client,
             profiles_path=Path("/path/to/.dbt/profiles.yml"),
             project_name="my_project",
+            profile_name="my_project",
             target_name="dev",
             import_db=True,
             disallow_edits=False,
@@ -303,6 +311,7 @@ def test_sync_database_external_url_prefix(
         client=client,
         profiles_path=Path("/path/to/.dbt/profiles.yml"),
         project_name="my_project",
+        profile_name="my_project",
         target_name="dev",
         import_db=True,
         disallow_edits=True,
@@ -339,6 +348,7 @@ def test_sync_database_existing(mocker: MockerFixture, fs: FakeFilesystem) -> No
         client=client,
         profiles_path=Path("/path/to/.dbt/profiles.yml"),
         project_name="my_project",
+        profile_name="my_project",
         target_name="dev",
         import_db=True,
         disallow_edits=False,
@@ -374,6 +384,7 @@ def test_sync_database_new_no_import(mocker: MockerFixture, fs: FakeFilesystem)
             client=client,
             profiles_path=Path("/path/to/.dbt/profiles.yml"),
             project_name="my_project",
+            profile_name="my_project",
             target_name="dev",
             import_db=False,
             disallow_edits=False,

tests/cli/superset/sync/dbt/lib_test.py

@@ -283,7 +283,7 @@ def test_load_profiles(monkeypatch: pytest.MonkeyPatch, fs: FakeFilesystem) -> N
     """,
     )
 
-    assert load_profiles(path, "jaffle_shop", "dev") == {
+    assert load_profiles(path, "jaffle_shop", "jaffle_shop", "dev") == {
         "jaffle_shop": {
             "outputs": {
                 "dev": {
@@ -306,7 +306,8 @@ def test_load_profiles(monkeypatch: pytest.MonkeyPatch, fs: FakeFilesystem) -> N
 
 
 def test_load_profiles_default_target(
-    monkeypatch: pytest.MonkeyPatch, fs: FakeFilesystem
+    monkeypatch: pytest.MonkeyPatch,
+    fs: FakeFilesystem,
 ) -> None:
     """
     Test ``load_profiles`` when no target is specified.
@@ -340,7 +341,7 @@ def test_load_profiles_default_target(
     """,
     )
 
-    assert load_profiles(path, "jaffle_shop", None) == {
+    assert load_profiles(path, "jaffle_shop", "jaffle_shop", None) == {
         "jaffle_shop": {
             "outputs": {
                 "dev": {