yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-18 12:27:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): fix dbt source platform when disable_dbt_node_creation is False (#3592)

Browse Source
This commit is contained in:
Remi 2021-11-17 17:49:48 -07:00 committed by GitHub
parent 3a9ef61147
commit a9754ebe83
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -396,11 +396,17 @@ def get_upstreams(
# create lineages for platform nodes otherwise, for dbt node, we connect it to another dbt node or a platform # create lineages for platform nodes otherwise, for dbt node, we connect it to another dbt node or a platform
# node. # node.
platform_value = DBT_PLATFORM platform_value = DBT_PLATFORM
if disable_dbt_node_creation: if disable_dbt_node_creation:
platform_value = target_platform platform_value = target_platform
else: else:
materialized = upstream_manifest_node.get("config", {}).get("materialized") materialized = upstream_manifest_node.get("config", {}).get("materialized")
if materialized in {"view", "table", "incremental"}: resource_type = upstream_manifest_node["resource_type"]
if (
materialized in {"view", "table", "incremental"}
or resource_type == "source"
):
platform_value = target_platform platform_value = target_platform
upstream_urns.append( upstream_urns.append(

20
metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View File

@ -101,7 +101,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -110,7 +110,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -119,7 +119,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -488,7 +488,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -497,7 +497,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -506,7 +506,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -515,7 +515,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -524,7 +524,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -533,7 +533,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -542,7 +542,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]

20
metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json
View File

@ -101,7 +101,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -110,7 +110,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -119,7 +119,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -488,7 +488,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -497,7 +497,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -506,7 +506,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -515,7 +515,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -524,7 +524,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -533,7 +533,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -542,7 +542,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]

20
metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json
View File

@ -101,7 +101,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -110,7 +110,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -119,7 +119,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -488,7 +488,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -497,7 +497,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -506,7 +506,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -515,7 +515,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -524,7 +524,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -533,7 +533,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -542,7 +542,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]