mirror of
https://github.com/datahub-project/datahub.git
synced 2025-11-03 20:27:50 +00:00
feat(ingest): ingest descriptions from dbt models (#2955)
This commit is contained in:
Kevin Hu
GitHub
parent
62d5306a28
commit
f82ea3abef
@ -55,6 +55,7 @@ class DBTConfig(ConfigModel):
|
||||
class DBTColumn:
|
||||
name: str
|
||||
comment: str
|
||||
description: str
|
||||
index: int
|
||||
data_type: str
|
||||
|
||||
@ -69,6 +70,7 @@ class DBTNode:
|
||||
schema: str
|
||||
name: str # name, identifier
|
||||
comment: str
|
||||
description: str
|
||||
|
||||
datahub_urn: str
|
||||
|
||||
@ -91,19 +93,22 @@ class DBTNode:
|
||||
return self.__class__.__name__ + str(tuple(sorted(fields))).replace("'", "")
|
||||
|
||||
|
||||
def get_columns(catalog_node: dict) -> List[DBTColumn]:
|
||||
def get_columns(catalog_node: dict, manifest_node: dict) -> List[DBTColumn]:
|
||||
columns = []
|
||||
|
||||
manifest_columns = manifest_node.get("columns", {})
|
||||
|
||||
raw_columns = catalog_node["columns"]
|
||||
|
||||
for key in raw_columns:
|
||||
raw_column = raw_columns[key]
|
||||
|
||||
dbtCol = DBTColumn(
|
||||
comment=raw_column["comment"],
|
||||
name=raw_column["name"],
|
||||
comment=raw_column.get("comment", ""),
|
||||
description=manifest_columns.get(key, {}).get("description", ""),
|
||||
data_type=raw_column["type"],
|
||||
index=raw_column["index"],
|
||||
name=raw_column["name"],
|
||||
)
|
||||
columns.append(dbtCol)
|
||||
return columns
|
||||
@ -123,20 +128,22 @@ def extract_dbt_entities(
|
||||
sources_by_id = {x["unique_id"]: x for x in sources_results}
|
||||
|
||||
dbt_entities = []
|
||||
for key, node in all_manifest_entities.items():
|
||||
for key, manifest_node in all_manifest_entities.items():
|
||||
# check if node pattern allowed based on config file
|
||||
if not node_type_pattern.allowed(node["resource_type"]):
|
||||
if not node_type_pattern.allowed(manifest_node["resource_type"]):
|
||||
continue
|
||||
|
||||
name = node["name"]
|
||||
name = manifest_node["name"]
|
||||
|
||||
if "identifier" in node and not load_catalog:
|
||||
name = node["identifier"]
|
||||
if "identifier" in manifest_node and not load_catalog:
|
||||
name = manifest_node["identifier"]
|
||||
|
||||
if node.get("alias") is not None:
|
||||
name = node["alias"]
|
||||
if manifest_node.get("alias") is not None:
|
||||
name = manifest_node["alias"]
|
||||
|
||||
comment = key
|
||||
# initialize comment to "" for consistency with descriptions
|
||||
# (since dbt null/undefined descriptions as "")
|
||||
comment = ""
|
||||
|
||||
if key in all_catalog_entities and all_catalog_entities[key]["metadata"].get(
|
||||
"comment"
|
||||
@ -146,11 +153,11 @@ def extract_dbt_entities(
|
||||
materialization = None
|
||||
upstream_urns = []
|
||||
|
||||
if "materialized" in node.get("config", {}).keys():
|
||||
if "materialized" in manifest_node.get("config", {}).keys():
|
||||
# It's a model
|
||||
materialization = node["config"]["materialized"]
|
||||
materialization = manifest_node["config"]["materialized"]
|
||||
upstream_urns = get_upstreams(
|
||||
node["depends_on"]["nodes"],
|
||||
manifest_node["depends_on"]["nodes"],
|
||||
all_manifest_entities,
|
||||
load_catalog,
|
||||
target_platform,
|
||||
@ -173,25 +180,26 @@ def extract_dbt_entities(
|
||||
|
||||
dbtNode = DBTNode(
|
||||
dbt_name=key,
|
||||
database=node["database"],
|
||||
schema=node["schema"],
|
||||
dbt_file_path=node["original_file_path"],
|
||||
node_type=node["resource_type"],
|
||||
database=manifest_node["database"],
|
||||
schema=manifest_node["schema"],
|
||||
dbt_file_path=manifest_node["original_file_path"],
|
||||
node_type=manifest_node["resource_type"],
|
||||
max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"),
|
||||
name=name,
|
||||
comment=comment,
|
||||
description=manifest_node.get("description", ""),
|
||||
upstream_urns=upstream_urns,
|
||||
materialization=materialization,
|
||||
catalog_type=catalog_type,
|
||||
columns=[],
|
||||
datahub_urn=get_urn_from_dbtNode(
|
||||
node["database"],
|
||||
node["schema"],
|
||||
manifest_node["database"],
|
||||
manifest_node["schema"],
|
||||
name,
|
||||
target_platform,
|
||||
environment,
|
||||
),
|
||||
meta=node.get("meta", {}),
|
||||
meta=manifest_node.get("meta", {}),
|
||||
)
|
||||
|
||||
# overwrite columns from catalog
|
||||
@ -207,7 +215,7 @@ def extract_dbt_entities(
|
||||
f"Entity {dbtNode.dbt_name} is in manifest but missing from catalog",
|
||||
)
|
||||
else:
|
||||
dbtNode.columns = get_columns(catalog_node)
|
||||
dbtNode.columns = get_columns(catalog_node, manifest_node)
|
||||
|
||||
else:
|
||||
dbtNode.columns = []
|
||||
@ -382,11 +390,21 @@ def get_schema_metadata(
|
||||
) -> SchemaMetadata:
|
||||
canonical_schema: List[SchemaField] = []
|
||||
for column in node.columns:
|
||||
|
||||
description = None
|
||||
|
||||
if column.comment and column.description:
|
||||
description = f"{platform} comment: {column.comment}\n\ndbt model description: {column.description}"
|
||||
elif column.comment:
|
||||
description = column.comment
|
||||
elif column.description:
|
||||
description = column.description
|
||||
|
||||
field = SchemaField(
|
||||
fieldPath=column.name,
|
||||
nativeDataType=column.data_type,
|
||||
type=get_column_type(report, node.dbt_name, column.data_type),
|
||||
description=column.comment,
|
||||
description=description,
|
||||
nullable=False, # TODO: actually autodetect this
|
||||
recursive=False,
|
||||
)
|
||||
@ -401,6 +419,8 @@ def get_schema_metadata(
|
||||
actor=actor,
|
||||
)
|
||||
|
||||
description = None
|
||||
|
||||
return SchemaMetadata(
|
||||
schemaName=node.dbt_name,
|
||||
platform=f"urn:li:dataPlatform:{platform}",
|
||||
@ -427,7 +447,6 @@ class DBTSource(Source):
|
||||
self.report = SourceReport()
|
||||
|
||||
def get_workunits(self) -> Iterable[MetadataWorkUnit]:
|
||||
platform = self.platform
|
||||
nodes = loadManifestAndCatalog(
|
||||
self.config.manifest_path,
|
||||
self.config.catalog_path,
|
||||
@ -446,8 +465,17 @@ class DBTSource(Source):
|
||||
aspects=[],
|
||||
)
|
||||
|
||||
description = None
|
||||
|
||||
if node.comment and node.description:
|
||||
description = f"{self.config.target_platform} comment: {node.comment}\n\ndbt model description: {node.description}"
|
||||
elif node.comment:
|
||||
description = node.comment
|
||||
elif node.description:
|
||||
description = node.description
|
||||
|
||||
dbt_properties = DatasetPropertiesClass(
|
||||
description=node.comment,
|
||||
description=description,
|
||||
customProperties=get_custom_properties(node),
|
||||
tags=[],
|
||||
)
|
||||
@ -458,7 +486,9 @@ class DBTSource(Source):
|
||||
dataset_snapshot.aspects.append(upstreams)
|
||||
|
||||
if self.config.load_schemas:
|
||||
schema_metadata = get_schema_metadata(self.report, node, platform)
|
||||
schema_metadata = get_schema_metadata(
|
||||
self.report, node, self.config.target_platform
|
||||
)
|
||||
dataset_snapshot.aspects.append(schema_metadata)
|
||||
|
||||
mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot)
|
||||
|
||||
@ -4,8 +4,8 @@
|
||||
"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json",
|
||||
"dbt_version": "0.19.1",
|
||||
"env": {},
|
||||
"generated_at": "2021-07-21T21:04:55.475026Z",
|
||||
"invocation_id": "edf3a531-d74f-4ca2-b8fe-fc17c9014e4d"
|
||||
"generated_at": "2021-07-26T16:12:02.512630Z",
|
||||
"invocation_id": "01d287bd-a8be-44f2-825e-c37570949bfb"
|
||||
},
|
||||
"nodes": {
|
||||
"model.sample_dbt.monthly_billing_with_cust": {
|
||||
@ -163,7 +163,7 @@
|
||||
"type": "integer"
|
||||
},
|
||||
"first_name": {
|
||||
"comment": null,
|
||||
"comment": "Actors column \u2013 from postgres",
|
||||
"index": 2,
|
||||
"name": "first_name",
|
||||
"type": "text"
|
||||
@ -182,7 +182,7 @@
|
||||
}
|
||||
},
|
||||
"metadata": {
|
||||
"comment": null,
|
||||
"comment": "Actors table \u2013 from postgres",
|
||||
"database": "pagila",
|
||||
"name": "actor",
|
||||
"owner": "postgres",
|
||||
|
||||
@ -3335,12 +3335,37 @@
|
||||
"selectors": {},
|
||||
"sources": {
|
||||
"source.sample_dbt.pagila.actor": {
|
||||
"columns": {},
|
||||
"columns": {
|
||||
"first_name": {
|
||||
"data_type": null,
|
||||
"description": "description for first_name from dbt",
|
||||
"meta": {},
|
||||
"name": "first_name",
|
||||
"quote": null,
|
||||
"tags": []
|
||||
},
|
||||
"last_name": {
|
||||
"data_type": null,
|
||||
"description": "description for last_name from dbt",
|
||||
"meta": {},
|
||||
"name": "last_name",
|
||||
"quote": null,
|
||||
"tags": []
|
||||
},
|
||||
"last_update": {
|
||||
"data_type": null,
|
||||
"description": "description for last_update from dbt",
|
||||
"meta": {},
|
||||
"name": "last_update",
|
||||
"quote": null,
|
||||
"tags": []
|
||||
}
|
||||
},
|
||||
"config": {
|
||||
"enabled": true
|
||||
},
|
||||
"database": "pagila",
|
||||
"description": "",
|
||||
"description": "description for actor table from dbt",
|
||||
"external": null,
|
||||
"fqn": [
|
||||
"sample_dbt",
|
||||
@ -3394,7 +3419,7 @@
|
||||
"enabled": true
|
||||
},
|
||||
"database": "pagila",
|
||||
"description": "",
|
||||
"description": "a user's address",
|
||||
"external": null,
|
||||
"fqn": [
|
||||
"sample_dbt",
|
||||
@ -3444,7 +3469,7 @@
|
||||
"enabled": true
|
||||
},
|
||||
"database": "pagila",
|
||||
"description": "",
|
||||
"description": "a user's category",
|
||||
"external": null,
|
||||
"fqn": [
|
||||
"sample_dbt",
|
||||
@ -3598,7 +3623,7 @@
|
||||
"enabled": true
|
||||
},
|
||||
"database": "pagila",
|
||||
"description": "",
|
||||
"description": "description for customer table from dbt",
|
||||
"external": null,
|
||||
"fqn": [
|
||||
"sample_dbt",
|
||||
@ -3857,7 +3882,7 @@
|
||||
"enabled": true
|
||||
},
|
||||
"database": "pagila",
|
||||
"description": "",
|
||||
"description": "a payment",
|
||||
"external": null,
|
||||
"fqn": [
|
||||
"sample_dbt",
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -13,7 +13,7 @@
|
||||
"dbt_file_path": "models/transform/customer_details.sql"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.customer_details",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -27,7 +27,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -36,7 +36,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -45,7 +45,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -54,7 +54,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "model.sample_dbt.customer_details",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -89,7 +89,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -100,7 +100,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.monthly_billing_with_cust",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -114,7 +114,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -123,7 +123,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -132,7 +132,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "model.sample_dbt.monthly_billing_with_cust",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -228,7 +228,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -239,7 +239,7 @@
|
||||
"catalog_type": "VIEW"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.payments_base",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -253,7 +253,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -262,7 +262,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -271,7 +271,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -280,7 +280,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -289,7 +289,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -298,7 +298,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -307,7 +307,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -316,7 +316,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "model.sample_dbt.payments_base",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -442,7 +442,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -453,7 +453,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.payments_by_customer_by_month",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -467,7 +467,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -476,7 +476,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "model.sample_dbt.payments_by_customer_by_month",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -557,7 +557,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -570,7 +570,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.actor",
|
||||
"description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -583,7 +583,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.actor",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -624,7 +624,7 @@
|
||||
"fieldPath": "first_name",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": null,
|
||||
"description": "postgres comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
@ -639,7 +639,7 @@
|
||||
"fieldPath": "last_name",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": null,
|
||||
"description": "description for last_name from dbt",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.StringType": {}
|
||||
@ -654,7 +654,7 @@
|
||||
"fieldPath": "last_update",
|
||||
"jsonPath": null,
|
||||
"nullable": false,
|
||||
"description": null,
|
||||
"description": "description for last_update from dbt",
|
||||
"type": {
|
||||
"type": {
|
||||
"com.linkedin.pegasus2avro.schema.TimeType": {}
|
||||
@ -679,7 +679,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -689,7 +689,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.address",
|
||||
"description": "a user's address",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -702,7 +702,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.address",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -858,7 +858,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -868,7 +868,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.category",
|
||||
"description": "a user's category",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -881,7 +881,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.category",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -962,7 +962,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -972,7 +972,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.city",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -985,7 +985,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.city",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1081,7 +1081,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -1094,7 +1094,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.country",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -1107,7 +1107,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.country",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1188,7 +1188,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -1198,7 +1198,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.customer",
|
||||
"description": "description for customer table from dbt",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -1211,7 +1211,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.customer",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1397,7 +1397,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -1407,7 +1407,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_01",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -1420,7 +1420,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.payment_p2020_01",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1546,7 +1546,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -1560,7 +1560,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_02",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -1573,7 +1573,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.payment_p2020_02",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1699,7 +1699,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -1709,7 +1709,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_03",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -1722,7 +1722,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.payment_p2020_03",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1848,7 +1848,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -1858,7 +1858,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_04",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -1871,7 +1871,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.payment_p2020_04",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -1997,7 +1997,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -2007,7 +2007,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_05",
|
||||
"description": "a payment",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -2020,7 +2020,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.payment_p2020_05",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
@ -2146,7 +2146,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -2156,7 +2156,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_06",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -2169,7 +2169,7 @@
|
||||
{
|
||||
"com.linkedin.pegasus2avro.schema.SchemaMetadata": {
|
||||
"schemaName": "source.sample_dbt.pagila.payment_p2020_06",
|
||||
"platform": "urn:li:dataPlatform:dbt",
|
||||
"platform": "urn:li:dataPlatform:postgres",
|
||||
"version": 0,
|
||||
"created": {
|
||||
"time": 0,
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -13,7 +13,7 @@
|
||||
"dbt_file_path": "models/transform/customer_details.sql"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.customer_details",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -27,7 +27,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -36,7 +36,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -45,7 +45,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -60,7 +60,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -71,7 +71,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.monthly_billing_with_cust",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -85,7 +85,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -94,7 +94,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -109,7 +109,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -120,7 +120,7 @@
|
||||
"catalog_type": "VIEW"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.payments_base",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -134,7 +134,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -143,7 +143,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -152,7 +152,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -161,7 +161,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -170,7 +170,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -179,7 +179,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
},
|
||||
{
|
||||
@ -188,7 +188,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -203,7 +203,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -214,7 +214,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "model.sample_dbt.payments_by_customer_by_month",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -228,7 +228,7 @@
|
||||
"actor": "urn:li:corpuser:unknown",
|
||||
"impersonator": null
|
||||
},
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
|
||||
"type": "TRANSFORMED"
|
||||
}
|
||||
]
|
||||
@ -243,7 +243,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -256,7 +256,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.actor",
|
||||
"description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -275,7 +275,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -285,7 +285,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.address",
|
||||
"description": "a user's address",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -304,7 +304,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -314,7 +314,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.category",
|
||||
"description": "a user's category",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -333,7 +333,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -343,7 +343,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.city",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -362,7 +362,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -375,7 +375,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.country",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -394,7 +394,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -404,7 +404,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.customer",
|
||||
"description": "description for customer table from dbt",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -423,7 +423,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -433,7 +433,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_01",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -452,7 +452,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -466,7 +466,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_02",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -485,7 +485,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -495,7 +495,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_03",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -514,7 +514,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -524,7 +524,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_04",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -543,7 +543,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -553,7 +553,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_05",
|
||||
"description": "a payment",
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
@ -572,7 +572,7 @@
|
||||
"auditHeader": null,
|
||||
"proposedSnapshot": {
|
||||
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
|
||||
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
|
||||
"aspects": [
|
||||
{
|
||||
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
|
||||
@ -582,7 +582,7 @@
|
||||
"catalog_type": "BASE TABLE"
|
||||
},
|
||||
"externalUrl": null,
|
||||
"description": "source.sample_dbt.pagila.payment_p2020_06",
|
||||
"description": null,
|
||||
"uri": null,
|
||||
"tags": []
|
||||
}
|
||||
|
||||
@ -28,7 +28,7 @@ class DbtTestConfig:
|
||||
self.manifest_path = f"{test_resources_dir}/dbt_manifest.json"
|
||||
self.catalog_path = f"{test_resources_dir}/dbt_catalog.json"
|
||||
self.sources_path = f"{test_resources_dir}/dbt_sources.json"
|
||||
self.target_platform = "dbt"
|
||||
self.target_platform = "postgres"
|
||||
|
||||
self.output_path = f"{tmp_path}/{output_file}"
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user