yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-31 21:03:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/dbt): add support for latest DBT version 1.3 (#6651)

Browse Source 
Co-authored-by: Matthieu Blais <matthieu.blais@tech.jago.com>
Co-authored-by: Harshal Sheth <hsheth2@gmail.com>
This commit is contained in:
Matthieu Blais 2022-12-07 08:03:24 +08:00 committed by GitHub
parent 438bbdee0f
commit 4e2dde84f6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 1944 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py
View File

@ -278,11 +278,11 @@ class DBTCloudSource(DBTSourceBase):
)
# The code fields are new in dbt 1.3, and replace the sql ones.
raw_sql = node["rawCode"] or node["rawSql"]
compiled_sql = node["compiledCode"] or node["compiledSql"]
raw_code = node["rawCode"] or node["rawSql"]
compiled_code = node["compiledCode"] or node["compiledSql"]
else:
raw_sql = None
compiled_sql = None
raw_code = None
compiled_code = None
max_loaded_at = None
if node["resourceType"] == "source":
@ -369,8 +369,9 @@ class DBTCloudSource(DBTSourceBase):
query_tag={}, # TODO: Get this from the dbt API.
tags=tags,
owner=owner,
raw_sql=raw_sql,
compiled_sql=compiled_sql,
language="sql", # TODO: dbt Cloud doesn't surface this
raw_code=raw_code,
compiled_code=compiled_code,
columns=columns,
test_info=test_info,
test_result=test_result,

23
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -354,7 +354,8 @@ class DBTNode:
alias: Optional[str] # alias if present
comment: str
description: str
raw_sql: Optional[str]
language: Optional[str]
raw_code: Optional[str]
dbt_adapter: str
dbt_name: str
@ -375,7 +376,7 @@ class DBTNode:
query_tag: Dict[str, Any] = field(default_factory=dict)
tags: List[str] = field(default_factory=list)
compiled_sql: Optional[str] = None
compiled_code: Optional[str] = None
test_info: Optional["DBTTest"] = None # only populated if node_type == 'test'
test_result: Optional["DBTTestResult"] = None
@ -410,7 +411,13 @@ def get_custom_properties(node: DBTNode) -> Dict[str, str]:
custom_properties = node.meta
# additional node attributes to extract to custom properties
node_attributes = ["node_type", "materialization", "dbt_file_path", "catalog_type"]
node_attributes = [
"node_type",
"materialization",
"dbt_file_path",
"catalog_type",
"language",
]
for attribute in node_attributes:
node_attribute_value = getattr(node, attribute)
@ -834,7 +841,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
mce_builder.make_schema_field_urn(upstream_urn, column_name)
],
nativeType=node.name,
logic=node.compiled_sql if node.compiled_sql else node.raw_sql,
logic=node.compiled_code if node.compiled_code else node.raw_code,
aggregation=AssertionStdAggregationClass._NATIVE_,
nativeParameters=string_map(kw_args),
),
@ -848,7 +855,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
dataset=upstream_urn,
scope=DatasetAssertionScopeClass.DATASET_ROWS,
operator=AssertionStdOperatorClass._NATIVE_,
logic=node.compiled_sql if node.compiled_sql else node.raw_sql,
logic=node.compiled_code if node.compiled_code else node.raw_code,
nativeType=node.name,
aggregation=AssertionStdAggregationClass._NATIVE_,
nativeParameters=string_map(kw_args),
@ -1023,7 +1030,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
aspects.append(upstream_lineage_class)
# add view properties aspect
if node.raw_sql:
if node.raw_code and node.language == "sql":
view_prop_aspect = self._create_view_properties_aspect(node)
aspects.append(view_prop_aspect)
@ -1157,11 +1164,11 @@ class DBTSourceBase(StatefulIngestionSourceBase):
def _create_view_properties_aspect(self, node: DBTNode) -> ViewPropertiesClass:
materialized = node.materialization in {"table", "incremental"}
# this function is only called when raw sql is present. assert is added to satisfy lint checks
assert node.raw_sql is not None
assert node.raw_code is not None
view_properties = ViewPropertiesClass(
materialized=materialized,
viewLanguage="SQL",
viewLogic=node.raw_sql,
viewLogic=node.raw_code,
)
return view_properties

11
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py
View File

@ -222,7 +222,12 @@ def extract_dbt_entities(
max_loaded_at=max_loaded_at,
comment=comment,
description=manifest_node.get("description", ""),
raw_sql=manifest_node.get("raw_sql"),
raw_code=manifest_node.get(
"raw_code", manifest_node.get("raw_sql")
), # Backward compatibility dbt <=v1.2
language=manifest_node.get(
"language", "sql"
), # Backward compatibility dbt <=v1.2
upstream_nodes=upstream_nodes,
materialization=materialization,
catalog_type=catalog_type,
@ -230,7 +235,9 @@ def extract_dbt_entities(
query_tag=query_tag_props,
tags=tags,
owner=owner,
compiled_sql=manifest_node.get("compiled_sql"),
compiled_code=manifest_node.get(
"compiled_code", manifest_node.get("compiled_sql")
), # Backward compatibility dbt <=v1.2
test_info=test_info,
)

15
metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json
View File

@ -25,6 +25,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -154,6 +155,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -263,6 +265,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -413,6 +416,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -618,6 +622,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -800,6 +805,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -922,6 +928,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -1059,6 +1066,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -1195,6 +1203,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -1401,6 +1410,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -1563,6 +1573,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -1735,6 +1746,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -1893,6 +1905,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -2051,6 +2064,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",
@ -2209,6 +2223,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v4.json",
"manifest_version": "1.0.3",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View File

@ -30,6 +30,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -204,6 +205,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -371,6 +373,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -585,6 +588,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -717,6 +721,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -878,6 +883,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1060,6 +1066,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1182,6 +1189,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1319,6 +1327,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1458,6 +1467,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1664,6 +1674,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1826,6 +1837,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2001,6 +2013,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2159,6 +2172,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2317,6 +2331,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2475,6 +2490,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

8
metadata-ingestion/tests/integration/dbt/dbt_stateful_tests_golden.json
View File

@ -25,6 +25,7 @@
"materialization": "view",
"dbt_file_path": "models/staging/stg_customers.sql",
"catalog_type": "view",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -155,6 +156,7 @@
"materialization": "view",
"dbt_file_path": "models/staging/stg_payments.sql",
"catalog_type": "view",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -297,6 +299,7 @@
"materialization": "view",
"dbt_file_path": "models/staging/stg_orders.sql",
"catalog_type": "view",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -439,6 +442,7 @@
"materialization": "seed",
"dbt_file_path": "seeds/raw_customers.csv",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -548,6 +552,7 @@
"materialization": "seed",
"dbt_file_path": "seeds/raw_orders.csv",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -669,6 +674,7 @@
"materialization": "seed",
"dbt_file_path": "seeds/raw_payments.csv",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -790,6 +796,7 @@
"materialization": "table",
"dbt_file_path": "models/customers.sql",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -990,6 +997,7 @@
"materialization": "table",
"dbt_file_path": "models/orders.sql",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",

114
metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
View File

@ -24,11 +24,12 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "customer_details",
"description": "",
@ -133,11 +134,12 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "an-aliased-view-for-monthly-billing",
"description": "",
@ -283,11 +285,12 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "an-aliased-view-for-payments",
"description": "",
@ -489,11 +492,12 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payments_by_customer_by_month",
"description": "",
@ -551,8 +555,8 @@
},
{
"fieldPath": "customer_id",
"nullable": false,
"description": "description for customer_id from dbt",
"nullable": false,
"type": {
"type": {
"com.linkedin.pegasus2avro.schema.NumberType": {}
@ -646,11 +650,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "actor",
"description": "description for actor table from dbt",
@ -800,11 +805,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "address",
"description": "a user's address",
@ -982,11 +988,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "category",
"description": "a user's category",
@ -1104,11 +1111,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "city",
"description": "",
@ -1241,11 +1249,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "country",
"description": "",
@ -1380,11 +1389,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "customer",
"description": "description for customer table from dbt",
@ -1586,11 +1596,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payment_p2020_01",
"description": "",
@ -1748,11 +1759,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payment_p2020_02",
"description": "",
@ -1923,11 +1935,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payment_p2020_03",
"description": "",
@ -2081,11 +2094,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payment_p2020_04",
"description": "",
@ -2239,11 +2253,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payment_p2020_05",
"description": "a payment",
@ -2397,11 +2412,12 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v6.json",
"manifest_version": "1.2.1",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v7.json",
"manifest_version": "1.3.0",
"manifest_adapter": "postgres",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "1.2.1"
"catalog_version": "1.3.0"
},
"name": "payment_p2020_06",
"description": "",

8
metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
View File

@ -25,6 +25,7 @@
"materialization": "view",
"dbt_file_path": "models/staging/stg_customers.sql",
"catalog_type": "view",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -155,6 +156,7 @@
"materialization": "view",
"dbt_file_path": "models/staging/stg_payments.sql",
"catalog_type": "view",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -297,6 +299,7 @@
"materialization": "view",
"dbt_file_path": "models/staging/stg_orders.sql",
"catalog_type": "view",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -439,6 +442,7 @@
"materialization": "seed",
"dbt_file_path": "seeds/raw_customers.csv",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -548,6 +552,7 @@
"materialization": "seed",
"dbt_file_path": "seeds/raw_orders.csv",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -669,6 +674,7 @@
"materialization": "seed",
"dbt_file_path": "seeds/raw_payments.csv",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -790,6 +796,7 @@
"materialization": "table",
"dbt_file_path": "models/customers.sql",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",
@ -990,6 +997,7 @@
"materialization": "table",
"dbt_file_path": "models/orders.sql",
"catalog_type": "table",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v5.json",
"manifest_version": "1.1.0",
"manifest_adapter": "bigquery",

15
metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
View File

@ -29,6 +29,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -168,6 +169,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -318,6 +320,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -532,6 +535,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -664,6 +668,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -822,6 +827,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1004,6 +1010,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1126,6 +1133,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1263,6 +1271,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1399,6 +1408,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1605,6 +1615,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1767,6 +1778,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1939,6 +1951,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2097,6 +2110,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2255,6 +2269,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
View File

@ -30,6 +30,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -169,6 +170,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -319,6 +321,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -533,6 +536,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -665,6 +669,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -823,6 +828,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1005,6 +1011,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1127,6 +1134,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1264,6 +1272,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1400,6 +1409,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1606,6 +1616,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1768,6 +1779,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1940,6 +1952,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2098,6 +2111,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2256,6 +2270,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2414,6 +2429,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
View File

@ -30,6 +30,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -169,6 +170,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -319,6 +321,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -533,6 +536,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -665,6 +669,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -823,6 +828,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1005,6 +1011,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1127,6 +1134,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1264,6 +1272,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1400,6 +1409,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1606,6 +1616,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1768,6 +1779,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1940,6 +1952,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2098,6 +2111,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2256,6 +2270,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2414,6 +2429,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

16
metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
View File

@ -30,6 +30,7 @@
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -169,6 +170,7 @@
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -319,6 +321,7 @@
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -533,6 +536,7 @@
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -665,6 +669,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -823,6 +828,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1005,6 +1011,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1127,6 +1134,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1264,6 +1272,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1400,6 +1409,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1606,6 +1616,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1768,6 +1779,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -1940,6 +1952,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2098,6 +2111,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2256,6 +2270,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",
@ -2414,6 +2429,7 @@
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"language": "sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"manifest_adapter": "postgres",

2
metadata-ingestion/tests/integration/dbt/sample_dbt_catalog.json
View File

@ -2,7 +2,7 @@
"errors": null,
"metadata": {
"dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"dbt_version": "1.2.1",
"dbt_version": "1.3.0",
"env": {},
"generated_at": "2021-06-19T21:38:36.384613Z",
"invocation_id": "just-some-random-id-2"

2072
metadata-ingestion/tests/integration/dbt/sample_dbt_manifest.json
View File

File diff suppressed because it is too large Load Diff

2
metadata-ingestion/tests/integration/dbt/sample_dbt_sources.json
View File

@ -2,7 +2,7 @@
"elapsed_time": 3.1415,
"metadata": {
"dbt_schema_version": "https://schemas.getdbt.com/dbt/sources/v3.json",
"dbt_version": "1.2.1",
"dbt_version": "1.3.0",
"env": {},
"generated_at": "2021-06-18T21:38:36.384613Z",
"invocation_id": "just-some-random-id"