yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-14 10:26:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): dbt - ensure platform_instance filled out only for dbt nodes (#5115)

Browse Source
This commit is contained in:
Gabe Lyons 2022-06-07 20:09:42 -07:00 committed by GitHub
parent 10b1333703
commit 538cfba526
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 296 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -431,7 +431,7 @@ def get_urn_from_dbtNode(
data_platform_instance: Optional[str], data_platform_instance: Optional[str],
) -> str: ) -> str:
db_fqn = get_db_fqn(database, schema, name) db_fqn = get_db_fqn(database, schema, name)
if data_platform_instance is not None: if data_platform_instance is not None and target_platform == DBT_PLATFORM:
db_fqn = f"{data_platform_instance}.{db_fqn}" db_fqn = f"{data_platform_instance}.{db_fqn}"
return mce_builder.make_dataset_urn(target_platform, db_fqn, env) return mce_builder.make_dataset_urn(target_platform, db_fqn, env)
@ -508,7 +508,7 @@ def get_upstreams(
name, name,
platform_value, platform_value,
environment, environment,
platform_instance if platform_value == DBT_PLATFORM else None, platform_instance,
) )
) )
return upstream_urns return upstream_urns
@ -921,7 +921,7 @@ class DBTSource(StatefulIngestionSourceBase):
node.name, node.name,
mce_platform, mce_platform,
self.config.env, self.config.env,
self.config.platform_instance if mce_platform == DBT_PLATFORM else None, self.config.platform_instance,
) )
self.save_checkpoint(node_datahub_urn) self.save_checkpoint(node_datahub_urn)
@ -1261,7 +1261,7 @@ class DBTSource(StatefulIngestionSourceBase):
self.config.target_platform, self.config.target_platform,
self.config.env, self.config.env,
self.config.disable_dbt_node_creation, self.config.disable_dbt_node_creation,
None, self.config.platform_instance,
) )
# if a node is of type source in dbt, its upstream lineage should have the corresponding table/view # if a node is of type source in dbt, its upstream lineage should have the corresponding table/view
@ -1274,7 +1274,7 @@ class DBTSource(StatefulIngestionSourceBase):
node.name, node.name,
self.config.target_platform, self.config.target_platform,
self.config.env, self.config.env,
None, self.config.platform_instance,
) )
) )
if upstream_urns: if upstream_urns:

356
metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
View File

File diff suppressed because it is too large Load Diff