fix(ingest): dbt - ensure platform_instance filled out only for dbt nodes (#5115)

This commit is contained in:
Gabe Lyons 2022-06-07 20:09:42 -07:00 committed by GitHub
parent 10b1333703
commit 538cfba526
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 296 additions and 70 deletions

View File

@ -431,7 +431,7 @@ def get_urn_from_dbtNode(
data_platform_instance: Optional[str],
) -> str:
db_fqn = get_db_fqn(database, schema, name)
if data_platform_instance is not None:
if data_platform_instance is not None and target_platform == DBT_PLATFORM:
db_fqn = f"{data_platform_instance}.{db_fqn}"
return mce_builder.make_dataset_urn(target_platform, db_fqn, env)
@ -508,7 +508,7 @@ def get_upstreams(
name,
platform_value,
environment,
platform_instance if platform_value == DBT_PLATFORM else None,
platform_instance,
)
)
return upstream_urns
@ -921,7 +921,7 @@ class DBTSource(StatefulIngestionSourceBase):
node.name,
mce_platform,
self.config.env,
self.config.platform_instance if mce_platform == DBT_PLATFORM else None,
self.config.platform_instance,
)
self.save_checkpoint(node_datahub_urn)
@ -1261,7 +1261,7 @@ class DBTSource(StatefulIngestionSourceBase):
self.config.target_platform,
self.config.env,
self.config.disable_dbt_node_creation,
None,
self.config.platform_instance,
)
# if a node is of type source in dbt, its upstream lineage should have the corresponding table/view
@ -1274,7 +1274,7 @@ class DBTSource(StatefulIngestionSourceBase):
node.name,
self.config.target_platform,
self.config.env,
None,
self.config.platform_instance,
)
)
if upstream_urns: