From a9754ebe83b6b73bc2bfbf49d9ebf5dbd2ca5a8f Mon Sep 17 00:00:00 2001 From: Remi Date: Wed, 17 Nov 2021 17:49:48 -0700 Subject: [PATCH] fix(ingest): fix dbt source platform when disable_dbt_node_creation is False (#3592) --- .../src/datahub/ingestion/source/dbt.py | 8 +++++++- .../dbt_enabled_with_schemas_mces_golden.json | 20 +++++++++---------- ...t_enabled_without_schemas_mces_golden.json | 20 +++++++++---------- ...thout_schemas_with_filter_mces_golden.json | 20 +++++++++---------- 4 files changed, 37 insertions(+), 31 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index bfb5170827..d155868144 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -396,11 +396,17 @@ def get_upstreams( # create lineages for platform nodes otherwise, for dbt node, we connect it to another dbt node or a platform # node. platform_value = DBT_PLATFORM + if disable_dbt_node_creation: platform_value = target_platform else: materialized = upstream_manifest_node.get("config", {}).get("materialized") - if materialized in {"view", "table", "incremental"}: + resource_type = upstream_manifest_node["resource_type"] + + if ( + materialized in {"view", "table", "incremental"} + or resource_type == "source" + ): platform_value = target_platform upstream_urns.append( diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index 551abb2c5d..2a2e690a0d 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -101,7 +101,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -110,7 +110,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -119,7 +119,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } ] @@ -488,7 +488,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -497,7 +497,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -506,7 +506,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -515,7 +515,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -524,7 +524,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -533,7 +533,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -542,7 +542,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ] diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json index 85c265f906..bff41efa37 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json @@ -101,7 +101,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -110,7 +110,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -119,7 +119,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } ] @@ -488,7 +488,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -497,7 +497,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -506,7 +506,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -515,7 +515,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -524,7 +524,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -533,7 +533,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -542,7 +542,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ] diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json index 0ea1438f1c..8a537d1243 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json @@ -101,7 +101,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -110,7 +110,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -119,7 +119,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", "type": "TRANSFORMED" } ] @@ -488,7 +488,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -497,7 +497,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -506,7 +506,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -515,7 +515,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -524,7 +524,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -533,7 +533,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -542,7 +542,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ]