yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-31 21:03:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): extract dbt meta fields (#2876)

Browse Source
This commit is contained in:
Kevin Hu 2021-07-13 14:58:25 -07:00 committed by GitHub
parent 9e3c520290
commit bc84c82a68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 33 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -83,6 +83,8 @@ class DBTNode:
columns: List[DBTColumn] = field(default_factory=list)
upstream_urns: List[str] = field(default_factory=list)
meta: Dict[str, Any] = field(default_factory=dict)
def __repr__(self):
fields = tuple("{}={}".format(k, v) for k, v in self.__dict__.items())
return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")
@ -177,6 +179,7 @@ def extract_dbt_entities(
target_platform,
environment,
),
meta=node.get("meta", {}),
)
# overwrite columns from catalog
@ -259,8 +262,11 @@ def get_urn_from_dbtNode(
def get_custom_properties(node: DBTNode) -> Dict[str, str]:
custom_properties = {}
# initialize custom properties to node's meta props
# (dbt-native node properties)
custom_properties = node.meta
# additional node attributes to extract to custom properties
node_attributes = ["node_type", "materialization", "dbt_file_path", "catalog_type"]
for attribute in node_attributes:

4
metadata-ingestion/tests/integration/dbt/dbt_catalog.json
View File

@ -4,8 +4,8 @@
"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"dbt_version": "0.19.1",
"env": {},
"generated_at": "2021-06-21T17:28:04.980076Z",
"invocation_id": "a878f3a5-ecf3-44b1-94ac-4334c92ca3f6"
"generated_at": "2021-07-13T18:38:17.650799Z",
"invocation_id": "a0290787-2b80-46c6-a3b7-bf882f654ff3"
},
"nodes": {
"model.sample_dbt.monthly_billing_with_cust": {

18
metadata-ingestion/tests/integration/dbt/dbt_manifest.json
View File

@ -3359,7 +3359,11 @@
"identifier": "actor",
"loaded_at_field": "last_update",
"loader": "",
"meta": {},
"meta": {
"model_maturity": "in dev",
"owner": "@alice",
"some_other_property": "test 1"
},
"name": "actor",
"original_file_path": "models/base.yml",
"package_name": "sample_dbt",
@ -3559,7 +3563,11 @@
"identifier": "country",
"loaded_at_field": "last_update",
"loader": "",
"meta": {},
"meta": {
"model_maturity": "in prod",
"owner": "@bob",
"some_other_property": "test 2"
},
"name": "country",
"original_file_path": "models/base.yml",
"package_name": "sample_dbt",
@ -3709,7 +3717,11 @@
"identifier": "payment_p2020_02",
"loaded_at_field": "payment_date",
"loader": "",
"meta": {},
"meta": {
"model_maturity": "in prod",
"owner": "@charles",
"some_other_property": "test 3"
},
"name": "payment_p2020_02",
"original_file_path": "models/base.yml",
"package_name": "sample_dbt",

9
metadata-ingestion/tests/integration/dbt/dbt_mces_golden.json
View File

@ -562,6 +562,9 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"model_maturity": "in dev",
"owner": "@alice",
"some_other_property": "test 1",
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE"
@ -1083,6 +1086,9 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"model_maturity": "in prod",
"owner": "@bob",
"some_other_property": "test 2",
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE"
@ -1545,6 +1551,9 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"model_maturity": "in prod",
"owner": "@charles",
"some_other_property": "test 3",
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE"