yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-14 03:26:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/dbt): only generate one subtype (#10615)

Browse Source
This commit is contained in:
Harshal Sheth 2024-05-29 17:11:34 -07:00 committed by GitHub
parent d78287cc57
commit db965d61ea
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 39 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -45,7 +45,6 @@ from datahub.ingestion.api.incremental_lineage_helper import (
from datahub.ingestion.api.source import MetadataWorkUnitProcessor from datahub.ingestion.api.source import MetadataWorkUnitProcessor
from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.source_helpers import auto_workunit
from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.api.workunit import MetadataWorkUnit
from datahub.ingestion.source.common.subtypes import DatasetSubTypes
from datahub.ingestion.source.dbt.dbt_tests import ( from datahub.ingestion.source.dbt.dbt_tests import (
DBTTest, DBTTest,
DBTTestResult, DBTTestResult,
@ -1739,12 +1738,6 @@ class DBTSourceBase(StatefulIngestionSourceBase):
return None return None
subtypes: List[str] = [node.node_type.capitalize()] subtypes: List[str] = [node.node_type.capitalize()]
if node.materialization == "table":
subtypes.append(DatasetSubTypes.TABLE)
if node.node_type == "model" or node.node_type == "snapshot":
# We need to add the view subtype so that the view properties tab shows up in the UI.
subtypes.append(DatasetSubTypes.VIEW)
return MetadataChangeProposalWrapper( return MetadataChangeProposalWrapper(
entityUrn=node_datahub_urn, entityUrn=node_datahub_urn,

14
metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -372,9 +371,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -606,8 +603,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -943,9 +939,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },

17
metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -337,9 +336,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -559,8 +556,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -888,9 +884,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -1107,8 +1101,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Snapshot", "Snapshot"
"View"
] ]
} }
}, },

17
metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
View File

@ -7,9 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -233,9 +231,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -478,8 +474,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -633,8 +628,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -800,8 +794,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },

25
metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -337,9 +336,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -565,7 +562,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)",
"type": "BATCH_SCHEDULED", "type": "BATCH_SCHEDULED",
"created": { "created": {
"time": 1643871600000, "time": 1663355198240,
"actor": "urn:li:corpuser:datahub" "actor": "urn:li:corpuser:datahub"
} }
} }
@ -685,8 +682,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -1020,7 +1016,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)",
"type": "BATCH_SCHEDULED", "type": "BATCH_SCHEDULED",
"created": { "created": {
"time": 1643871600000, "time": 1663355198240,
"actor": "urn:li:corpuser:datahub" "actor": "urn:li:corpuser:datahub"
} }
} }
@ -1144,9 +1140,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -1349,7 +1343,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)",
"type": "BATCH_SCHEDULED", "type": "BATCH_SCHEDULED",
"created": { "created": {
"time": 1643871600000, "time": 1663355198240,
"actor": "urn:li:corpuser:datahub" "actor": "urn:li:corpuser:datahub"
} }
} }
@ -1468,8 +1462,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Snapshot", "Snapshot"
"View"
] ]
} }
}, },
@ -1873,7 +1866,7 @@
"name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)",
"type": "BATCH_SCHEDULED", "type": "BATCH_SCHEDULED",
"created": { "created": {
"time": 1643871600000, "time": 1663355198240,
"actor": "urn:li:corpuser:datahub" "actor": "urn:li:corpuser:datahub"
} }
} }

14
metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -336,9 +335,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -552,8 +549,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -889,9 +885,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },

14
metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -338,9 +337,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -555,8 +552,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -893,9 +889,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },

14
metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -337,9 +336,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -553,8 +550,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -890,9 +886,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },

14
metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
View File

@ -7,8 +7,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -337,9 +336,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },
@ -553,8 +550,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"View"
] ]
} }
}, },
@ -890,9 +886,7 @@
"aspect": { "aspect": {
"json": { "json": {
"typeNames": [ "typeNames": [
"Model", "Model"
"Table",
"View"
] ]
} }
}, },