Allow ignoring of certain upstream lineage platforms (#15)

* Allow ignoring of certain upstream lineage platforms

* Handle whitespace

* Remove lineage_platform_instance

metadata-ingestion/src/datahub/ingestion/source/tableau.py

@@ -353,9 +353,9 @@ class TableauConfig(
         description="[Experimental] Whether to extract lineage from unsupported custom sql queries using SQL parsing",
     )
 
-    lineage_platform_instance: str = Field(
-        default="uk",
-        description="Platform instance for mapping upstream lineage URNs.",
+    ignore_upstream_lineage_platforms: Optional[str] = Field(
+        default="",
+        description="Comma separated list of platforms to not ingest upstream lineage for",
     )
 
     # pre = True because we want to take some decision before pydantic initialize the configuration to default values
@@ -499,6 +499,12 @@ def __init__(
         # when emitting custom SQL data sources.
         self.custom_sql_ids_being_used: List[str] = []
 
+        if self.config.ignore_upstream_lineage_platforms:
+            self.ignore_upstream_lineage_platforms = [
+                x.strip()
+                for x in (self.config.ignore_upstream_lineage_platforms.split(","))
+            ]
+
         self._authenticate()
 
     def close(self) -> None:
@@ -975,6 +981,15 @@ def get_upstream_tables(
         # Same table urn can be used when setting fine grained lineage,
         table_id_to_urn: Dict[str, str] = {}
         for table in tables:
+            if (
+                table.get(tableau_constant.CONNECTION_TYPE, "")
+                in self.ignore_upstream_lineage_platforms
+            ):
+                logger.debug(
+                    f"Skipping upstream table {table[tableau_constant.ID]}, ignoring upstream platform {table.get(tableau_constant.CONNECTION_TYPE, '')}"
+                )
+                continue
+
             # skip upstream tables when there is no column info when retrieving datasource
             # Lineage and Schema details for these will be taken care in self.emit_custom_sql_datasources()
             num_tbl_cols: Optional[int] = table.get(c.COLUMNS_CONNECTION) and table[
@@ -1004,7 +1019,6 @@ def get_upstream_tables(
                 self.config.env,
                 self.config.platform_instance_map,
                 self.config.lineage_overrides,
-                self.config.lineage_platform_instance
             )
             table_id_to_urn[table[c.ID]] = table_urn
 

metadata-ingestion/src/datahub/ingestion/source/tableau_common.py

@@ -543,7 +543,6 @@ def get_fully_qualified_table_name(
     upstream_db: str,
     schema: str,
     table_name: str,
-    lineage_platform_instance: Optional[str] = None,
 ) -> str:
     if platform == "athena":
         upstream_db = ""
@@ -576,9 +575,6 @@ def get_fully_qualified_table_name(
             fully_qualified_table_name.split(".")[-3:]
         )
 
-    if lineage_platform_instance:
-        fully_qualified_table_name = f"{lineage_platform_instance}.{fully_qualified_table_name}"
-
     return fully_qualified_table_name