yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-30 03:18:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): dbt aliases (#2927)

Browse Source
This commit is contained in:
Kevin Hu 2021-07-22 07:57:14 -07:00 committed by GitHub
parent 99790d9052
commit 84a1486b89
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 30 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -133,6 +133,9 @@ def extract_dbt_entities(
if "identifier" in node and not load_catalog:
name = node["identifier"]
if node.get("alias") is not None:
name = node["alias"]
comment = key
if key in all_catalog_entities and all_catalog_entities[key]["metadata"].get(
@ -161,7 +164,7 @@ def extract_dbt_entities(
if catalog_node is None:
report.report_warning(
key,
f"Entity {name} is in manifest but missing from catalog",
f"Entity {key} ({name}) is in manifest but missing from catalog",
)
else:
@ -305,6 +308,9 @@ def get_upstreams(
else:
name = all_nodes[upstream]["name"]
if "alias" in all_nodes[upstream]:
name = all_nodes[upstream]["alias"]
upstream_urns.append(
get_urn_from_dbtNode(
all_nodes[upstream]["database"],

8
metadata-ingestion/tests/integration/dbt/dbt_catalog.json
View File

@ -4,8 +4,8 @@
"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"dbt_version": "0.19.1",
"env": {},
"generated_at": "2021-07-20T17:22:41.553024Z",
"invocation_id": "b95e2c51-f4af-42ae-bc78-197497edf22c"
"generated_at": "2021-07-21T21:04:55.475026Z",
"invocation_id": "edf3a531-d74f-4ca2-b8fe-fc17c9014e4d"
},
"nodes": {
"model.sample_dbt.monthly_billing_with_cust": {
@ -38,7 +38,7 @@
"metadata": {
"comment": null,
"database": "pagila",
"name": "monthly_billing_with_cust",
"name": "an-aliased-view-for-monthly-billing",
"owner": "postgres",
"schema": "dbt_postgres",
"type": "BASE TABLE"
@ -96,7 +96,7 @@
"metadata": {
"comment": null,
"database": "pagila",
"name": "payments_base",
"name": "an-aliased-view-for-payments",
"owner": "postgres",
"schema": "dbt_postgres",
"type": "VIEW"

24
metadata-ingestion/tests/integration/dbt/dbt_manifest.json
View File

@ -3054,17 +3054,17 @@
}
},
"model.sample_dbt.monthly_billing_with_cust": {
"alias": "monthly_billing_with_cust",
"alias": "an-aliased-view-for-monthly-billing",
"build_path": "target/compiled/sample_dbt/models/billing/monthly_billing_with_cust.sql",
"checksum": {
"checksum": "96468f0564c87d3dfb75f80f0bf1dac169f67137dd46eeefdb2a9939a03fad13",
"checksum": "3f13dfa22648e3bb3e52082de053591b33b07ca028216db3b12a1a09d6bf747a",
"name": "sha256"
},
"columns": {},
"compiled": true,
"compiled_sql": "\n\nwith __dbt__CTE__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)SELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n \"pagila\".\"dbt_postgres\".\"payments_by_customer_by_month\" pbc\n left outer join __dbt__CTE__customer_details cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month",
"config": {
"alias": null,
"alias": "an-aliased-view-for-monthly-billing",
"column_types": {},
"database": null,
"enabled": true,
@ -3109,7 +3109,7 @@
"package_name": "sample_dbt",
"patch_path": null,
"path": "billing/monthly_billing_with_cust.sql",
"raw_sql": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month",
"raw_sql": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.billing_month",
"refs": [
[
"payments_by_customer_by_month"
@ -3118,7 +3118,7 @@
"customer_details"
]
],
"relation_name": "\"pagila\".\"dbt_postgres\".\"monthly_billing_with_cust\"",
"relation_name": "\"pagila\".\"dbt_postgres\".\"an-aliased-view-for-monthly-billing\"",
"resource_type": "model",
"root_path": "/some-path/sample-dbt",
"schema": "dbt_postgres",
@ -3126,21 +3126,22 @@
"tags": [],
"unique_id": "model.sample_dbt.monthly_billing_with_cust",
"unrendered_config": {
"alias": "an-aliased-view-for-monthly-billing",
"materialized": "table"
}
},
"model.sample_dbt.payments_base": {
"alias": "payments_base",
"alias": "an-aliased-view-for-payments",
"build_path": "target/compiled/sample_dbt/models/base/payments_base.sql",
"checksum": {
"checksum": "697370dca0dcbac9bb193630534ee3a54c9503488607b564a6390924bf4baa70",
"checksum": "11032ac45a744ecfe350f4ba5d7cbdd8b47bc83f0b39474dd0e7839888768dd3",
"name": "sha256"
},
"columns": {},
"compiled": true,
"compiled_sql": "\n\nwith payments as (\n\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_06\"\n)\n\nselect *\nfrom payments",
"config": {
"alias": null,
"alias": "an-aliased-view-for-payments",
"column_types": {},
"database": null,
"enabled": true,
@ -3185,9 +3186,9 @@
"package_name": "sample_dbt",
"patch_path": null,
"path": "base/payments_base.sql",
"raw_sql": "{{ config(\n materialized=\"view\",\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments",
"raw_sql": "{{ config(\n materialized=\"view\",\n alias='an-aliased-view-for-payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments",
"refs": [],
"relation_name": "\"pagila\".\"dbt_postgres\".\"payments_base\"",
"relation_name": "\"pagila\".\"dbt_postgres\".\"an-aliased-view-for-payments\"",
"resource_type": "model",
"root_path": "/some-path/sample-dbt",
"schema": "dbt_postgres",
@ -3224,6 +3225,7 @@
"tags": [],
"unique_id": "model.sample_dbt.payments_base",
"unrendered_config": {
"alias": "an-aliased-view-for-payments",
"materialized": "view"
}
},
@ -3236,7 +3238,7 @@
},
"columns": {},
"compiled": true,
"compiled_sql": "\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n \"pagila\".\"dbt_postgres\".\"payments_base\"\nGROUP BY\n billing_month,\n customer_id",
"compiled_sql": "\n\nSELECT\n date_trunc('month', payment_date) as \"billing_month\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n \"pagila\".\"dbt_postgres\".\"an-aliased-view-for-payments\"\nGROUP BY\n billing_month,\n customer_id",
"config": {
"alias": null,
"column_types": {},

6
metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json
View File

@ -89,7 +89,7 @@
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.monthly_billing_with_cust,PROD)",
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -228,7 +228,7 @@
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_base,PROD)",
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -467,7 +467,7 @@
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_base,PROD)",
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"type": "TRANSFORMED"
}
]

6
metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json
View File

@ -60,7 +60,7 @@
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.monthly_billing_with_cust,PROD)",
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -109,7 +109,7 @@
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_base,PROD)",
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -228,7 +228,7 @@
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_base,PROD)",
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"type": "TRANSFORMED"
}
]