Skip to content

Commit

Permalink
fix(ingest): fix inferred column lineage for upstream Athena tables
Browse files Browse the repository at this point in the history
  • Loading branch information
maiarareinaldo committed Mar 5, 2024
1 parent 4970750 commit 8f2a7d4
Showing 1 changed file with 4 additions and 1 deletion.
5 changes: 4 additions & 1 deletion metadata-ingestion/src/datahub/utilities/sqlglot_lineage.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,13 @@ def as_sqlglot_table(self) -> sqlglot.exp.Table:
def qualified(
self,
dialect: str,
platform: str,
default_db: Optional[str] = None,
default_schema: Optional[str] = None,
) -> "_TableName":
database = self.database or default_db
if platform == "athena":
database = "awsdatacatalog"
db_schema = self.db_schema or default_schema

return _TableName(
Expand Down Expand Up @@ -1028,7 +1031,7 @@ def _sqlglot_lineage_inner(
# For select statements, qualification will be a no-op. For other statements, this
# is where the qualification actually happens.
qualified_table = table.qualified(
dialect=dialect, default_db=default_db, default_schema=default_schema
dialect=dialect, default_db=default_db, default_schema=default_schema, platform=schema_resolver.platform
)

urn, schema_info = schema_resolver.resolve_table(qualified_table)
Expand Down

0 comments on commit 8f2a7d4

Please sign in to comment.