diff --git a/src/preset_cli/cli/superset/sync/dbt/command.py b/src/preset_cli/cli/superset/sync/dbt/command.py index b37cac2c..ec49571f 100644 --- a/src/preset_cli/cli/superset/sync/dbt/command.py +++ b/src/preset_cli/cli/superset/sync/dbt/command.py @@ -3,6 +3,8 @@ """ import os.path +import sys +import warnings from pathlib import Path from typing import Optional, Tuple @@ -22,8 +24,8 @@ @click.command() -@click.argument("manifest", type=click.Path(exists=True, resolve_path=True)) -@click.option("--project", help="Name of the dbt project", default="default") +@click.argument("file", type=click.Path(exists=True, resolve_path=True)) +@click.option("--project", help="Name of the dbt project", default=None) @click.option("--target", help="Target name", default=None) @click.option( "--profiles", @@ -63,8 +65,8 @@ @click.pass_context def dbt_core( # pylint: disable=too-many-arguments, too-many-locals ctx: click.core.Context, - manifest: str, - project: str, + file: str, + project: Optional[str], target: Optional[str], select: Tuple[str, ...], exclude: Tuple[str, ...], @@ -84,11 +86,40 @@ def dbt_core( # pylint: disable=too-many-arguments, too-many-locals if profiles is None: profiles = os.path.expanduser("~/.dbt/profiles.yml") + file_path = Path(file) + if file_path.name == "manifest.json": + warnings.warn( + ( + "Passing the manifest.json file is deprecated. " + "Please pass the dbt_project.yml file instead." + ), + category=DeprecationWarning, + stacklevel=2, + ) + manifest = file_path + profile = project = project or "default" + elif file_path.name == "dbt_project.yml": + with open(file_path, encoding="utf-8") as input_: + dbt_project = yaml.load(input_, Loader=yaml.SafeLoader) + + manifest = file_path.parent / dbt_project["target-path"] / "manifest.json" + profile = dbt_project["profile"] + project = project or dbt_project["name"] + else: + click.echo( + click.style( + "FILE should be either manifest.json or dbt_project.yml", + fg="bright_red", + ), + ) + sys.exit(1) + try: database = sync_database( client, Path(profiles), project, + profile, target, import_db, disallow_edits, diff --git a/src/preset_cli/cli/superset/sync/dbt/databases.py b/src/preset_cli/cli/superset/sync/dbt/databases.py index 99087619..b14ecf16 100644 --- a/src/preset_cli/cli/superset/sync/dbt/databases.py +++ b/src/preset_cli/cli/superset/sync/dbt/databases.py @@ -19,6 +19,7 @@ def sync_database( # pylint: disable=too-many-locals, too-many-arguments client: SupersetClient, profiles_path: Path, project_name: str, + profile_name: str, target_name: Optional[str], import_db: bool, disallow_edits: bool, # pylint: disable=unused-argument @@ -29,7 +30,7 @@ def sync_database( # pylint: disable=too-many-locals, too-many-arguments """ base_url = URL(external_url_prefix) if external_url_prefix else None - profiles = load_profiles(profiles_path, project_name, target_name) + profiles = load_profiles(profiles_path, project_name, profile_name, target_name) project = profiles[project_name] outputs = project["outputs"] if target_name is None: diff --git a/src/preset_cli/cli/superset/sync/dbt/lib.py b/src/preset_cli/cli/superset/sync/dbt/lib.py index 92cc5272..a1925459 100644 --- a/src/preset_cli/cli/superset/sync/dbt/lib.py +++ b/src/preset_cli/cli/superset/sync/dbt/lib.py @@ -232,7 +232,10 @@ class Target(TypedDict): def load_profiles( - path: Path, project_name: str, target_name: Optional[str], + path: Path, + project_name: str, + profile_name: str, + target_name: Optional[str], ) -> Dict[str, Any]: """ Load the file and apply Jinja2 templating. @@ -240,9 +243,9 @@ def load_profiles( with open(path, encoding="utf-8") as input_: profiles = yaml.load(input_, Loader=yaml.SafeLoader) - if project_name not in profiles: - raise Exception(f"Project {project_name} not found in {path}") - project = profiles[project_name] + if profile_name not in profiles: + raise Exception(f"Project {profile_name} not found in {path}") + project = profiles[profile_name] outputs = project["outputs"] if target_name is None: target_name = project["target"] @@ -259,6 +262,7 @@ def load_profiles( context = { "env_var": env_var, "project_name": project_name, + "profile_name": profile_name, "target": target, } diff --git a/tests/cli/superset/sync/dbt/command_test.py b/tests/cli/superset/sync/dbt/command_test.py index a3a466bb..0c8e0952 100644 --- a/tests/cli/superset/sync/dbt/command_test.py +++ b/tests/cli/superset/sync/dbt/command_test.py @@ -7,6 +7,7 @@ from pathlib import Path import pytest +import yaml from click.testing import CliRunner from pyfakefs.fake_filesystem import FakeFilesystem from pytest_mock import MockerFixture @@ -67,6 +68,7 @@ def test_dbt_core(mocker: MockerFixture, fs: FakeFilesystem) -> None: client, profiles, "default", + "default", None, False, False, @@ -107,6 +109,92 @@ def test_dbt_core(mocker: MockerFixture, fs: FakeFilesystem) -> None: sync_exposures.assert_called_with(client, exposures, sync_datasets()) +def test_dbt_core_dbt_project(mocker: MockerFixture, fs: FakeFilesystem) -> None: + """ + Test the ``dbt-core`` command with a ``dbt_project.yml`` file. + """ + root = Path("/path/to/root") + fs.create_dir(root) + dbt_project = root / "default/dbt_project.yml" + fs.create_file( + dbt_project, + contents=yaml.dump( + { + "name": "my_project", + "profile": "default", + "target-path": "target", + }, + ), + ) + manifest = root / "default/target/manifest.json" + fs.create_file(manifest, contents=manifest_contents) + profiles = root / ".dbt/profiles.yml" + fs.create_file(profiles) + + SupersetClient = mocker.patch( + "preset_cli.cli.superset.sync.dbt.command.SupersetClient", + ) + client = SupersetClient() + mocker.patch("preset_cli.cli.superset.main.UsernamePasswordAuth") + sync_database = mocker.patch( + "preset_cli.cli.superset.sync.dbt.command.sync_database", + ) + + runner = CliRunner() + result = runner.invoke( + superset_cli, + [ + "https://superset.example.org/", + "sync", + "dbt-core", + str(dbt_project), + "--profiles", + str(profiles), + ], + catch_exceptions=False, + ) + assert result.exit_code == 0 + sync_database.assert_called_with( + client, + profiles, + "my_project", + "default", + None, + False, + False, + "", + ) + + +def test_dbt_core_invalid_argument(mocker: MockerFixture, fs: FakeFilesystem) -> None: + """ + Test the ``dbt-core`` command with an invalid argument. + """ + root = Path("/path/to/root") + fs.create_dir(root) + profiles = root / ".dbt/profiles.yml" + fs.create_file(profiles) + wrong = root / "wrong" + fs.create_file(wrong) + + mocker.patch("preset_cli.cli.superset.sync.dbt.command.SupersetClient") + mocker.patch("preset_cli.cli.superset.main.UsernamePasswordAuth") + + runner = CliRunner() + result = runner.invoke( + superset_cli, + [ + "https://superset.example.org/", + "sync", + "dbt-core", + str(wrong), + ], + catch_exceptions=False, + ) + assert result.exit_code == 1 + assert result.output == "FILE should be either manifest.json or dbt_project.yml\n" + + def test_dbt(mocker: MockerFixture, fs: FakeFilesystem) -> None: """ Test the ``dbt`` command. @@ -158,6 +246,7 @@ def test_dbt(mocker: MockerFixture, fs: FakeFilesystem) -> None: client, profiles, "default", + "default", None, False, False, @@ -279,6 +368,7 @@ def test_dbt_core_default_profile(mocker: MockerFixture, fs: FakeFilesystem) -> client, profiles, "default", + "default", None, False, False, diff --git a/tests/cli/superset/sync/dbt/databases_test.py b/tests/cli/superset/sync/dbt/databases_test.py index 28aa29d6..4ce6f147 100644 --- a/tests/cli/superset/sync/dbt/databases_test.py +++ b/tests/cli/superset/sync/dbt/databases_test.py @@ -33,6 +33,7 @@ def test_sync_database_new(mocker: MockerFixture, fs: FakeFilesystem) -> None: client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=True, disallow_edits=False, @@ -48,7 +49,8 @@ def test_sync_database_new(mocker: MockerFixture, fs: FakeFilesystem) -> None: def test_sync_database_new_default_target( - mocker: MockerFixture, fs: FakeFilesystem + mocker: MockerFixture, + fs: FakeFilesystem, ) -> None: """ Test ``sync_database`` when we want to import a new DB using the default target. @@ -68,6 +70,7 @@ def test_sync_database_new_default_target( client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name=None, import_db=True, disallow_edits=False, @@ -121,6 +124,7 @@ def test_sync_database_new_custom_sqlalchemy_uri( client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=True, disallow_edits=False, @@ -177,6 +181,7 @@ def test_sync_database_env_var( client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=True, disallow_edits=False, @@ -206,7 +211,8 @@ def test_sync_database_no_project(mocker: MockerFixture, fs: FakeFilesystem) -> sync_database( client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), - project_name="my_other_project", + project_name="my_project", + profile_name="my_other_project", target_name="dev", import_db=True, disallow_edits=False, @@ -234,6 +240,7 @@ def test_sync_database_no_target(mocker: MockerFixture, fs: FakeFilesystem) -> N client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="prod", import_db=True, disallow_edits=False, @@ -273,6 +280,7 @@ def test_sync_database_multiple_databases( client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=True, disallow_edits=False, @@ -303,6 +311,7 @@ def test_sync_database_external_url_prefix( client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=True, disallow_edits=True, @@ -339,6 +348,7 @@ def test_sync_database_existing(mocker: MockerFixture, fs: FakeFilesystem) -> No client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=True, disallow_edits=False, @@ -374,6 +384,7 @@ def test_sync_database_new_no_import(mocker: MockerFixture, fs: FakeFilesystem) client=client, profiles_path=Path("/path/to/.dbt/profiles.yml"), project_name="my_project", + profile_name="my_project", target_name="dev", import_db=False, disallow_edits=False, diff --git a/tests/cli/superset/sync/dbt/lib_test.py b/tests/cli/superset/sync/dbt/lib_test.py index 72a480ac..bd128cef 100644 --- a/tests/cli/superset/sync/dbt/lib_test.py +++ b/tests/cli/superset/sync/dbt/lib_test.py @@ -283,7 +283,7 @@ def test_load_profiles(monkeypatch: pytest.MonkeyPatch, fs: FakeFilesystem) -> N """, ) - assert load_profiles(path, "jaffle_shop", "dev") == { + assert load_profiles(path, "jaffle_shop", "jaffle_shop", "dev") == { "jaffle_shop": { "outputs": { "dev": { @@ -306,7 +306,8 @@ def test_load_profiles(monkeypatch: pytest.MonkeyPatch, fs: FakeFilesystem) -> N def test_load_profiles_default_target( - monkeypatch: pytest.MonkeyPatch, fs: FakeFilesystem + monkeypatch: pytest.MonkeyPatch, + fs: FakeFilesystem, ) -> None: """ Test ``load_profiles`` when no target is specified. @@ -340,7 +341,7 @@ def test_load_profiles_default_target( """, ) - assert load_profiles(path, "jaffle_shop", None) == { + assert load_profiles(path, "jaffle_shop", "jaffle_shop", None) == { "jaffle_shop": { "outputs": { "dev": {