Skip to content

Commit

Permalink
first pass: microbatch
Browse files Browse the repository at this point in the history
  • Loading branch information
MichelleArk committed Sep 19, 2024
1 parent 3fc624c commit 0b8f111
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,9 @@
{%- elif strategy == 'insert_overwrite' -%}
{#-- insert statements don't like CTEs, so support them via a temp view #}
{{ get_insert_overwrite_sql(source, target, existing) }}
{%- elif strategy == 'microbatch' -%}
{#-- insert statements don't like CTEs, so support them via a temp view #}
{{ get_insert_overwrite_sql(source, target, existing) }}
{%- elif strategy == 'merge' -%}
{#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}
{{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}
Expand Down
17 changes: 17 additions & 0 deletions tests/functional/adapter/incremental_strategies/test_microbatch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
import pytest

from dbt.tests.adapter.incremental.test_incremental_microbatch import (
BaseMicrobatch,
)

# No requirement for a unique_id for spark microbatch!
_microbatch_model_no_unique_id_sql = """
{{ config(materialized='incremental', incremental_strategy='microbatch', event_time='event_time', batch_size='day') }}
select * from {{ ref('input_model') }}
"""


class TestMicrobatch(BaseMicrobatch):
@pytest.fixture(scope="class")
def microbatch_model_sql(self) -> str:
return _microbatch_model_no_unique_id_sql

0 comments on commit 0b8f111

Please sign in to comment.