yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-27 18:07:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/dbt): include dbt unique_id in properties (#7737)

Browse Source
This commit is contained in:
Harshal Sheth 2023-04-04 13:37:13 +05:30 committed by GitHub
parent 3609167086
commit 1634edaf25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 143 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -419,17 +419,16 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
custom_properties = node.meta
# additional node attributes to extract to custom properties
node_attributes = [
"node_type",
"materialization",
"dbt_file_path",
"catalog_type",
"language",
]
for attribute in node_attributes:
node_attribute_value = getattr(node, attribute)
node_attributes = {
"node_type": node.node_type,
"materialization": node.materialization,
"dbt_file_path": node.dbt_file_path,
"catalog_type": node.catalog_type,
"language": node.language,
"dbt_unique_id": node.dbt_name,
}
for attribute, node_attribute_value in node_attributes.items():
if node_attribute_value is not None:
custom_properties[attribute] = node_attribute_value

46
metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View File

@ -35,6 +35,7 @@
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -214,6 +215,7 @@
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -386,6 +388,7 @@
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -605,6 +608,7 @@
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -741,6 +745,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -906,6 +911,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1092,6 +1098,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1218,6 +1225,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1359,6 +1367,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1502,6 +1511,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1712,6 +1722,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1878,6 +1889,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2057,6 +2069,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2219,6 +2232,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2381,6 +2395,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2543,6 +2558,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2733,6 +2749,21 @@
"runId": "dbt-test-with-schemas-dbt-enabled"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:dbt:has_pii_test",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "dbt:has_pii_test"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:dbt:my_query_tag",
@ -2777,20 +2808,5 @@
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:dbt:has_pii_test",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "dbt:has_pii_test"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled"
}
}
]

47
metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
View File

@ -29,6 +29,7 @@
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -151,6 +152,7 @@
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -306,6 +308,7 @@
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -517,6 +520,7 @@
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -680,6 +684,7 @@
"dbt_file_path": "snapshots/customer_snapshot.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "snapshot.sample_dbt.customer_snapshot",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -948,6 +953,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -1107,6 +1113,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -1293,6 +1300,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -1419,6 +1427,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -1560,6 +1569,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -1703,6 +1713,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -1913,6 +1924,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -2079,6 +2091,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -2258,6 +2271,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -2420,6 +2434,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -2582,6 +2597,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -2744,6 +2760,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
@ -2933,21 +2950,6 @@
"runId": "dbt-column-meta-mapping"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:dbt:tag_from_dbt",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "dbt:tag_from_dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:dbt:sensitive",
@ -2962,5 +2964,20 @@
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping"
}
},
{
"entityType": "tag",
"entityUrn": "urn:li:tag:dbt:tag_from_dbt",
"changeType": "UPSERT",
"aspectName": "tagKey",
"aspect": {
"json": {
"name": "dbt:tag_from_dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping"
}
}
]

8
metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
View File

@ -30,6 +30,7 @@
"dbt_file_path": "models/customers.sql",
"catalog_type": "table",
"language": "sql",
"dbt_unique_id": "model.jaffle_shop.customers",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -235,6 +236,7 @@
"dbt_file_path": "models/orders.sql",
"catalog_type": "table",
"language": "sql",
"dbt_unique_id": "model.jaffle_shop.orders",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -459,6 +461,7 @@
"dbt_file_path": "models/staging/stg_customers.sql",
"catalog_type": "view",
"language": "sql",
"dbt_unique_id": "model.jaffle_shop.stg_customers",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -594,6 +597,7 @@
"dbt_file_path": "models/staging/stg_orders.sql",
"catalog_type": "view",
"language": "sql",
"dbt_unique_id": "model.jaffle_shop.stg_orders",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -741,6 +745,7 @@
"dbt_file_path": "models/staging/stg_payments.sql",
"catalog_type": "view",
"language": "sql",
"dbt_unique_id": "model.jaffle_shop.stg_payments",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -887,6 +892,7 @@
"dbt_file_path": "seeds/raw_customers.csv",
"catalog_type": "table",
"language": "sql",
"dbt_unique_id": "seed.jaffle_shop.raw_customers",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -1000,6 +1006,7 @@
"dbt_file_path": "seeds/raw_orders.csv",
"catalog_type": "table",
"language": "sql",
"dbt_unique_id": "seed.jaffle_shop.raw_orders",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -1125,6 +1132,7 @@
"dbt_file_path": "seeds/raw_payments.csv",
"catalog_type": "table",
"language": "sql",
"dbt_unique_id": "seed.jaffle_shop.raw_payments",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",

15
metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
View File

@ -34,6 +34,7 @@
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -178,6 +179,7 @@
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -333,6 +335,7 @@
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -552,6 +555,7 @@
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -688,6 +692,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -850,6 +855,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1036,6 +1042,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1162,6 +1169,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1303,6 +1311,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1443,6 +1452,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1653,6 +1663,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1819,6 +1830,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1995,6 +2007,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2157,6 +2170,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2319,6 +2333,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
View File

@ -35,6 +35,7 @@
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -179,6 +180,7 @@
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -334,6 +336,7 @@
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -553,6 +556,7 @@
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -689,6 +693,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -851,6 +856,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1037,6 +1043,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1163,6 +1170,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1304,6 +1312,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1444,6 +1453,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1654,6 +1664,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1820,6 +1831,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1996,6 +2008,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2158,6 +2171,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2320,6 +2334,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2482,6 +2497,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
View File

@ -35,6 +35,7 @@
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -179,6 +180,7 @@
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -334,6 +336,7 @@
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -553,6 +556,7 @@
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -689,6 +693,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -851,6 +856,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1037,6 +1043,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1163,6 +1170,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1304,6 +1312,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1444,6 +1453,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1654,6 +1664,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1820,6 +1831,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1996,6 +2008,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2158,6 +2171,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2320,6 +2334,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2482,6 +2497,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
View File

@ -35,6 +35,7 @@
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.customer_details",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -179,6 +180,7 @@
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -334,6 +336,7 @@
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_base",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -553,6 +556,7 @@
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -689,6 +693,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.actor",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -851,6 +856,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.address",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1037,6 +1043,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.category",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1163,6 +1170,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.city",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1304,6 +1312,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.country",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1444,6 +1453,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.customer",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1654,6 +1664,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1820,6 +1831,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1996,6 +2008,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2158,6 +2171,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2320,6 +2334,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2482,6 +2497,7 @@
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",