yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-29 12:13:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): ingest descriptions from dbt models (#2955)

Browse Source
This commit is contained in:
Kevin Hu 2021-07-26 10:47:25 -07:00 committed by GitHub
parent 62d5306a28
commit f82ea3abef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 201 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -55,6 +55,7 @@ class DBTConfig(ConfigModel):
class DBTColumn: class DBTColumn:
name: str name: str
comment: str comment: str
description: str
index: int index: int
data_type: str data_type: str
@ -69,6 +70,7 @@ class DBTNode:
schema: str schema: str
name: str # name, identifier name: str # name, identifier
comment: str comment: str
description: str
datahub_urn: str datahub_urn: str
@ -91,19 +93,22 @@ class DBTNode:
return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "") return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")
def get_columns(catalog_node: dict) -> List[DBTColumn]: def get_columns(catalog_node: dict, manifest_node: dict) -> List[DBTColumn]:
columns = [] columns = []
manifest_columns = manifest_node.get("columns", {})
raw_columns = catalog_node["columns"] raw_columns = catalog_node["columns"]
for key in raw_columns: for key in raw_columns:
raw_column = raw_columns[key] raw_column = raw_columns[key]
dbtCol = DBTColumn( dbtCol = DBTColumn(
comment=raw_column["comment"], name=raw_column["name"],
comment=raw_column.get("comment", ""),
description=manifest_columns.get(key, {}).get("description", ""),
data_type=raw_column["type"], data_type=raw_column["type"],
index=raw_column["index"], index=raw_column["index"],
name=raw_column["name"],
) )
columns.append(dbtCol) columns.append(dbtCol)
return columns return columns
@ -123,20 +128,22 @@ def extract_dbt_entities(
sources_by_id = {x["unique_id"]: x for x in sources_results} sources_by_id = {x["unique_id"]: x for x in sources_results}
dbt_entities = [] dbt_entities = []
for key, node in all_manifest_entities.items(): for key, manifest_node in all_manifest_entities.items():
# check if node pattern allowed based on config file # check if node pattern allowed based on config file
if not node_type_pattern.allowed(node["resource_type"]): if not node_type_pattern.allowed(manifest_node["resource_type"]):
continue continue
name = node["name"] name = manifest_node["name"]
if "identifier" in node and not load_catalog: if "identifier" in manifest_node and not load_catalog:
name = node["identifier"] name = manifest_node["identifier"]
if node.get("alias") is not None: if manifest_node.get("alias") is not None:
name = node["alias"] name = manifest_node["alias"]
comment = key # initialize comment to "" for consistency with descriptions
# (since dbt null/undefined descriptions as "")
comment = ""
if key in all_catalog_entities and all_catalog_entities[key]["metadata"].get( if key in all_catalog_entities and all_catalog_entities[key]["metadata"].get(
"comment" "comment"
@ -146,11 +153,11 @@ def extract_dbt_entities(
materialization = None materialization = None
upstream_urns = [] upstream_urns = []
if "materialized" in node.get("config", {}).keys(): if "materialized" in manifest_node.get("config", {}).keys():
# It's a model # It's a model
materialization = node["config"]["materialized"] materialization = manifest_node["config"]["materialized"]
upstream_urns = get_upstreams( upstream_urns = get_upstreams(
node["depends_on"]["nodes"], manifest_node["depends_on"]["nodes"],
all_manifest_entities, all_manifest_entities,
load_catalog, load_catalog,
target_platform, target_platform,
@ -173,25 +180,26 @@ def extract_dbt_entities(
dbtNode = DBTNode( dbtNode = DBTNode(
dbt_name=key, dbt_name=key,
database=node["database"], database=manifest_node["database"],
schema=node["schema"], schema=manifest_node["schema"],
dbt_file_path=node["original_file_path"], dbt_file_path=manifest_node["original_file_path"],
node_type=node["resource_type"], node_type=manifest_node["resource_type"],
max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"), max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"),
name=name, name=name,
comment=comment, comment=comment,
description=manifest_node.get("description", ""),
upstream_urns=upstream_urns, upstream_urns=upstream_urns,
materialization=materialization, materialization=materialization,
catalog_type=catalog_type, catalog_type=catalog_type,
columns=[], columns=[],
datahub_urn=get_urn_from_dbtNode( datahub_urn=get_urn_from_dbtNode(
node["database"], manifest_node["database"],
node["schema"], manifest_node["schema"],
name, name,
target_platform, target_platform,
environment, environment,
), ),
meta=node.get("meta", {}), meta=manifest_node.get("meta", {}),
) )
# overwrite columns from catalog # overwrite columns from catalog
@ -207,7 +215,7 @@ def extract_dbt_entities(
f"Entity {dbtNode.dbt_name} is in manifest but missing from catalog", f"Entity {dbtNode.dbt_name} is in manifest but missing from catalog",
) )
else: else:
dbtNode.columns = get_columns(catalog_node) dbtNode.columns = get_columns(catalog_node, manifest_node)
else: else:
dbtNode.columns = [] dbtNode.columns = []
@ -382,11 +390,21 @@ def get_schema_metadata(
) -> SchemaMetadata: ) -> SchemaMetadata:
canonical_schema: List[SchemaField] = [] canonical_schema: List[SchemaField] = []
for column in node.columns: for column in node.columns:
description = None
if column.comment and column.description:
description = f"{platform} comment: {column.comment}\n\ndbt model description: {column.description}"
elif column.comment:
description = column.comment
elif column.description:
description = column.description
field = SchemaField( field = SchemaField(
fieldPath=column.name, fieldPath=column.name,
nativeDataType=column.data_type, nativeDataType=column.data_type,
type=get_column_type(report, node.dbt_name, column.data_type), type=get_column_type(report, node.dbt_name, column.data_type),
description=column.comment, description=description,
nullable=False, # TODO: actually autodetect this nullable=False, # TODO: actually autodetect this
recursive=False, recursive=False,
) )
@ -401,6 +419,8 @@ def get_schema_metadata(
actor=actor, actor=actor,
) )
description = None
return SchemaMetadata( return SchemaMetadata(
schemaName=node.dbt_name, schemaName=node.dbt_name,
platform=f"urn:li:dataPlatform:{platform}", platform=f"urn:li:dataPlatform:{platform}",
@ -427,7 +447,6 @@ class DBTSource(Source):
self.report = SourceReport() self.report = SourceReport()
def get_workunits(self) -> Iterable[MetadataWorkUnit]: def get_workunits(self) -> Iterable[MetadataWorkUnit]:
platform = self.platform
nodes = loadManifestAndCatalog( nodes = loadManifestAndCatalog(
self.config.manifest_path, self.config.manifest_path,
self.config.catalog_path, self.config.catalog_path,
@ -446,8 +465,17 @@ class DBTSource(Source):
aspects=[], aspects=[],
) )
description = None
if node.comment and node.description:
description = f"{self.config.target_platform} comment: {node.comment}\n\ndbt model description: {node.description}"
elif node.comment:
description = node.comment
elif node.description:
description = node.description
dbt_properties = DatasetPropertiesClass( dbt_properties = DatasetPropertiesClass(
description=node.comment, description=description,
customProperties=get_custom_properties(node), customProperties=get_custom_properties(node),
tags=[], tags=[],
) )
@ -458,7 +486,9 @@ class DBTSource(Source):
dataset_snapshot.aspects.append(upstreams) dataset_snapshot.aspects.append(upstreams)
if self.config.load_schemas: if self.config.load_schemas:
schema_metadata = get_schema_metadata(self.report, node, platform) schema_metadata = get_schema_metadata(
self.report, node, self.config.target_platform
)
dataset_snapshot.aspects.append(schema_metadata) dataset_snapshot.aspects.append(schema_metadata)
mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)

8
metadata-ingestion/tests/integration/dbt/dbt_catalog.json
View File

@ -4,8 +4,8 @@
"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", "dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"dbt_version": "0.19.1", "dbt_version": "0.19.1",
"env": {}, "env": {},
"generated_at": "2021-07-21T21:04:55.475026Z", "generated_at": "2021-07-26T16:12:02.512630Z",
"invocation_id": "edf3a531-d74f-4ca2-b8fe-fc17c9014e4d" "invocation_id": "01d287bd-a8be-44f2-825e-c37570949bfb"
}, },
"nodes": { "nodes": {
"model.sample_dbt.monthly_billing_with_cust": { "model.sample_dbt.monthly_billing_with_cust": {
@ -163,7 +163,7 @@
"type": "integer" "type": "integer"
}, },
"first_name": { "first_name": {
"comment": null, "comment": "Actors column \u2013 from postgres",
"index": 2, "index": 2,
"name": "first_name", "name": "first_name",
"type": "text" "type": "text"
@ -182,7 +182,7 @@
} }
}, },
"metadata": { "metadata": {
"comment": null, "comment": "Actors table \u2013 from postgres",
"database": "pagila", "database": "pagila",
"name": "actor", "name": "actor",
"owner": "postgres", "owner": "postgres",

37
metadata-ingestion/tests/integration/dbt/dbt_manifest.json
View File

@ -3335,12 +3335,37 @@
"selectors": {}, "selectors": {},
"sources": { "sources": {
"source.sample_dbt.pagila.actor": { "source.sample_dbt.pagila.actor": {
"columns": {}, "columns": {
"first_name": {
"data_type": null,
"description": "description for first_name from dbt",
"meta": {},
"name": "first_name",
"quote": null,
"tags": []
},
"last_name": {
"data_type": null,
"description": "description for last_name from dbt",
"meta": {},
"name": "last_name",
"quote": null,
"tags": []
},
"last_update": {
"data_type": null,
"description": "description for last_update from dbt",
"meta": {},
"name": "last_update",
"quote": null,
"tags": []
}
},
"config": { "config": {
"enabled": true "enabled": true
}, },
"database": "pagila", "database": "pagila",
"description": "", "description": "description for actor table from dbt",
"external": null, "external": null,
"fqn": [ "fqn": [
"sample_dbt", "sample_dbt",
@ -3394,7 +3419,7 @@
"enabled": true "enabled": true
}, },
"database": "pagila", "database": "pagila",
"description": "", "description": "a user's address",
"external": null, "external": null,
"fqn": [ "fqn": [
"sample_dbt", "sample_dbt",
@ -3444,7 +3469,7 @@
"enabled": true "enabled": true
}, },
"database": "pagila", "database": "pagila",
"description": "", "description": "a user's category",
"external": null, "external": null,
"fqn": [ "fqn": [
"sample_dbt", "sample_dbt",
@ -3598,7 +3623,7 @@
"enabled": true "enabled": true
}, },
"database": "pagila", "database": "pagila",
"description": "", "description": "description for customer table from dbt",
"external": null, "external": null,
"fqn": [ "fqn": [
"sample_dbt", "sample_dbt",
@ -3857,7 +3882,7 @@
"enabled": true "enabled": true
}, },
"database": "pagila", "database": "pagila",
"description": "", "description": "a payment",
"external": null, "external": null,
"fqn": [ "fqn": [
"sample_dbt", "sample_dbt",

128
metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json
View File

@ -3,7 +3,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -13,7 +13,7 @@
"dbt_file_path": "models/transform/customer_details.sql" "dbt_file_path": "models/transform/customer_details.sql"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.customer_details", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -27,7 +27,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -36,7 +36,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -45,7 +45,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -54,7 +54,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "model.sample_dbt.customer_details", "schemaName": "model.sample_dbt.customer_details",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -89,7 +89,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -100,7 +100,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.monthly_billing_with_cust", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -114,7 +114,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -123,7 +123,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -132,7 +132,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "model.sample_dbt.monthly_billing_with_cust", "schemaName": "model.sample_dbt.monthly_billing_with_cust",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -228,7 +228,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -239,7 +239,7 @@
"catalog_type": "VIEW" "catalog_type": "VIEW"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.payments_base", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -253,7 +253,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -262,7 +262,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -271,7 +271,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -280,7 +280,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -289,7 +289,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -298,7 +298,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -307,7 +307,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -316,7 +316,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "model.sample_dbt.payments_base", "schemaName": "model.sample_dbt.payments_base",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -442,7 +442,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -453,7 +453,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.payments_by_customer_by_month", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -467,7 +467,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -476,7 +476,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "model.sample_dbt.payments_by_customer_by_month", "schemaName": "model.sample_dbt.payments_by_customer_by_month",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -557,7 +557,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -570,7 +570,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.actor", "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -583,7 +583,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.actor", "schemaName": "source.sample_dbt.pagila.actor",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -624,7 +624,7 @@
"fieldPath": "first_name", "fieldPath": "first_name",
"jsonPath": null, "jsonPath": null,
"nullable": false, "nullable": false,
"description": null, "description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt",
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.StringType": {} "com.linkedin.pegasus2avro.schema.StringType": {}
@ -639,7 +639,7 @@
"fieldPath": "last_name", "fieldPath": "last_name",
"jsonPath": null, "jsonPath": null,
"nullable": false, "nullable": false,
"description": null, "description": "description for last_name from dbt",
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.StringType": {} "com.linkedin.pegasus2avro.schema.StringType": {}
@ -654,7 +654,7 @@
"fieldPath": "last_update", "fieldPath": "last_update",
"jsonPath": null, "jsonPath": null,
"nullable": false, "nullable": false,
"description": null, "description": "description for last_update from dbt",
"type": { "type": {
"type": { "type": {
"com.linkedin.pegasus2avro.schema.TimeType": {} "com.linkedin.pegasus2avro.schema.TimeType": {}
@ -679,7 +679,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -689,7 +689,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.address", "description": "a user's address",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -702,7 +702,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.address", "schemaName": "source.sample_dbt.pagila.address",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -858,7 +858,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -868,7 +868,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.category", "description": "a user's category",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -881,7 +881,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.category", "schemaName": "source.sample_dbt.pagila.category",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -962,7 +962,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -972,7 +972,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.city", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -985,7 +985,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.city", "schemaName": "source.sample_dbt.pagila.city",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1081,7 +1081,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -1094,7 +1094,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.country", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -1107,7 +1107,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.country", "schemaName": "source.sample_dbt.pagila.country",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1188,7 +1188,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -1198,7 +1198,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.customer", "description": "description for customer table from dbt",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -1211,7 +1211,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.customer", "schemaName": "source.sample_dbt.pagila.customer",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1397,7 +1397,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -1407,7 +1407,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_01", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -1420,7 +1420,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.payment_p2020_01", "schemaName": "source.sample_dbt.pagila.payment_p2020_01",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1546,7 +1546,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -1560,7 +1560,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_02", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -1573,7 +1573,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.payment_p2020_02", "schemaName": "source.sample_dbt.pagila.payment_p2020_02",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1699,7 +1699,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -1709,7 +1709,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_03", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -1722,7 +1722,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.payment_p2020_03", "schemaName": "source.sample_dbt.pagila.payment_p2020_03",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1848,7 +1848,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -1858,7 +1858,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_04", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -1871,7 +1871,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.payment_p2020_04", "schemaName": "source.sample_dbt.pagila.payment_p2020_04",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -1997,7 +1997,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -2007,7 +2007,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_05", "description": "a payment",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -2020,7 +2020,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.payment_p2020_05", "schemaName": "source.sample_dbt.pagila.payment_p2020_05",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,
@ -2146,7 +2146,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -2156,7 +2156,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_06", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -2169,7 +2169,7 @@
{ {
"com.linkedin.pegasus2avro.schema.SchemaMetadata": { "com.linkedin.pegasus2avro.schema.SchemaMetadata": {
"schemaName": "source.sample_dbt.pagila.payment_p2020_06", "schemaName": "source.sample_dbt.pagila.payment_p2020_06",
"platform": "urn:li:dataPlatform:dbt", "platform": "urn:li:dataPlatform:postgres",
"version": 0, "version": 0,
"created": { "created": {
"time": 0, "time": 0,

90
metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json
View File

@ -3,7 +3,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -13,7 +13,7 @@
"dbt_file_path": "models/transform/customer_details.sql" "dbt_file_path": "models/transform/customer_details.sql"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.customer_details", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -27,7 +27,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -36,7 +36,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -45,7 +45,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -60,7 +60,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -71,7 +71,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.monthly_billing_with_cust", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -85,7 +85,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -94,7 +94,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -109,7 +109,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -120,7 +120,7 @@
"catalog_type": "VIEW" "catalog_type": "VIEW"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.payments_base", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -134,7 +134,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -143,7 +143,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -152,7 +152,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -161,7 +161,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -170,7 +170,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -179,7 +179,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
}, },
{ {
@ -188,7 +188,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -203,7 +203,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -214,7 +214,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "model.sample_dbt.payments_by_customer_by_month", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -228,7 +228,7 @@
"actor": "urn:li:corpuser:unknown", "actor": "urn:li:corpuser:unknown",
"impersonator": null "impersonator": null
}, },
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)", "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"type": "TRANSFORMED" "type": "TRANSFORMED"
} }
] ]
@ -243,7 +243,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -256,7 +256,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.actor", "description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -275,7 +275,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -285,7 +285,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.address", "description": "a user's address",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -304,7 +304,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -314,7 +314,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.category", "description": "a user's category",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -333,7 +333,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -343,7 +343,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.city", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -362,7 +362,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -375,7 +375,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.country", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -394,7 +394,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -404,7 +404,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.customer", "description": "description for customer table from dbt",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -423,7 +423,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -433,7 +433,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_01", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -452,7 +452,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -466,7 +466,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_02", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -485,7 +485,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -495,7 +495,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_03", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -514,7 +514,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -524,7 +524,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_04", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -543,7 +543,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -553,7 +553,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_05", "description": "a payment",
"uri": null, "uri": null,
"tags": [] "tags": []
} }
@ -572,7 +572,7 @@
"auditHeader": null, "auditHeader": null,
"proposedSnapshot": { "proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"aspects": [ "aspects": [
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
@ -582,7 +582,7 @@
"catalog_type": "BASE TABLE" "catalog_type": "BASE TABLE"
}, },
"externalUrl": null, "externalUrl": null,
"description": "source.sample_dbt.pagila.payment_p2020_06", "description": null,
"uri": null, "uri": null,
"tags": [] "tags": []
} }

2
metadata-ingestion/tests/integration/dbt/test_dbt.py
View File

@ -28,7 +28,7 @@ class DbtTestConfig:
self.manifest_path = f"{test_resources_dir}/dbt_manifest.json" self.manifest_path = f"{test_resources_dir}/dbt_manifest.json"
self.catalog_path = f"{test_resources_dir}/dbt_catalog.json" self.catalog_path = f"{test_resources_dir}/dbt_catalog.json"
self.sources_path = f"{test_resources_dir}/dbt_sources.json" self.sources_path = f"{test_resources_dir}/dbt_sources.json"
self.target_platform = "dbt" self.target_platform = "postgres"
self.output_path = f"{tmp_path}/{output_file}" self.output_path = f"{tmp_path}/{output_file}"