Skip to content

Commit

Permalink
feat(ingest/glue): add catalog alias to the urn
Browse files Browse the repository at this point in the history
  • Loading branch information
maiarareinaldo committed Mar 4, 2024
1 parent 290ff47 commit f468462
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion metadata-ingestion/src/datahub/ingestion/source/aws/glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@


DEFAULT_PLATFORM = "glue"
AWS_DATA_CATALOG = "awsdatacatalog"
VALID_PLATFORMS = [DEFAULT_PLATFORM, "athena"]


Expand Down Expand Up @@ -161,6 +162,10 @@ class GlueSourceConfig(
stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = Field(
default=None, description=""
)
catalog_alias: str = Field(
default=AWS_DATA_CATALOG,
description="The catalog alias to be used in the dataset URN.",
)

def is_profiling_enabled(self) -> bool:
return self.profiling is not None and is_profiling_enabled(
Expand Down Expand Up @@ -940,7 +945,7 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
for table in tables:
database_name = table["DatabaseName"]
table_name = table["Name"]
full_table_name = f"{database_name}.{table_name}"
full_table_name = f"{self.source_config.catalog_alias}.{database_name}.{table_name}"
self.report.report_table_scanned()
if not self.source_config.database_pattern.allowed(
database_name
Expand Down

0 comments on commit f468462

Please sign in to comment.