yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-07 06:13:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(dbt-ingestion): add ability to skip specific models (#3340)

Browse Source
This commit is contained in:
AndreasTA-AW 2021-10-26 07:31:19 +02:00 committed by GitHub
parent 43f1aeb538
commit 561c04bcf8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 831 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
metadata-ingestion/source_docs/dbt.md
View File

@ -30,6 +30,8 @@ This plugin pulls metadata from dbt's artifact files:
- Prefix added to tags during ingestion. - Prefix added to tags during ingestion.
- node_type_pattern: - node_type_pattern:
- Use this filter to exclude and include node types using allow or deny method - Use this filter to exclude and include node types using allow or deny method
- node_name_pattern
- Use this filter to exclude and include node by names using allow or deny method
## Quickstart recipe ## Quickstart recipe
@ -71,6 +73,10 @@ Note that a `.` is used to denote nested fields in the YAML recipe.
| `node_type_pattern.allow` | | | List of regex patterns for dbt nodes to include in ingestion. | | `node_type_pattern.allow` | | | List of regex patterns for dbt nodes to include in ingestion. |
| `node_type_pattern.deny` | | | List of regex patterns for dbt nodes to exclude from ingestion. | | `node_type_pattern.deny` | | | List of regex patterns for dbt nodes to exclude from ingestion. |
| `node_type_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | | `node_type_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. |
| `node_name_pattern.allow` | | | List of regex patterns for dbt model names to include in ingestion. |
| `node_name_pattern.deny` | | | List of regex patterns for dbt model names to exclude from ingestion. |
| `node_name_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. |
## Compatibility ## Compatibility

22
metadata-ingestion/src/datahub/ingestion/source/dbt.py
View File

@ -59,6 +59,7 @@ class DBTConfig(ConfigModel):
use_identifiers: bool = False use_identifiers: bool = False
node_type_pattern: AllowDenyPattern = AllowDenyPattern.allow_all() node_type_pattern: AllowDenyPattern = AllowDenyPattern.allow_all()
tag_prefix: str = "dbt:" tag_prefix: str = "dbt:"
node_name_pattern: AllowDenyPattern = AllowDenyPattern.allow_all()
@dataclass @dataclass
@ -146,6 +147,7 @@ def extract_dbt_entities(
environment: str, environment: str,
node_type_pattern: AllowDenyPattern, node_type_pattern: AllowDenyPattern,
report: SourceReport, report: SourceReport,
node_name_pattern: AllowDenyPattern,
) -> List[DBTNode]: ) -> List[DBTNode]:
sources_by_id = {x["unique_id"]: x for x in sources_results} sources_by_id = {x["unique_id"]: x for x in sources_results}
@ -157,13 +159,15 @@ def extract_dbt_entities(
continue continue
name = manifest_node["name"] name = manifest_node["name"]
if "identifier" in manifest_node and use_identifiers: if "identifier" in manifest_node and use_identifiers:
name = manifest_node["identifier"] name = manifest_node["identifier"]
if manifest_node.get("alias") is not None: if manifest_node.get("alias") is not None:
name = manifest_node["alias"] name = manifest_node["alias"]
if not node_name_pattern.allowed(key):
continue
# initialize comment to "" for consistency with descriptions # initialize comment to "" for consistency with descriptions
# (since dbt null/undefined descriptions as "") # (since dbt null/undefined descriptions as "")
comment = "" comment = ""
@ -201,7 +205,10 @@ def extract_dbt_entities(
catalog_type = all_catalog_entities[key]["metadata"]["type"] catalog_type = all_catalog_entities[key]["metadata"]["type"]
meta = manifest_node.get("meta", {}) meta = manifest_node.get("meta", {})
owner = meta.get("owner") owner = meta.get("owner")
if owner is None:
owner = manifest_node.get("config", {}).get("meta", {}).get("owner")
tags = manifest_node.get("tags", []) tags = manifest_node.get("tags", [])
tags = [tag_prefix + tag for tag in tags] tags = [tag_prefix + tag for tag in tags]
@ -210,10 +217,10 @@ def extract_dbt_entities(
dbt_name=key, dbt_name=key,
database=manifest_node["database"], database=manifest_node["database"],
schema=manifest_node["schema"], schema=manifest_node["schema"],
name=name,
dbt_file_path=manifest_node["original_file_path"], dbt_file_path=manifest_node["original_file_path"],
node_type=manifest_node["resource_type"], node_type=manifest_node["resource_type"],
max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"), max_loaded_at=sources_by_id.get(key, {}).get("max_loaded_at"),
name=name,
comment=comment, comment=comment,
description=manifest_node.get("description", ""), description=manifest_node.get("description", ""),
upstream_urns=upstream_urns, upstream_urns=upstream_urns,
@ -266,6 +273,7 @@ def loadManifestAndCatalog(
environment: str, environment: str,
node_type_pattern: AllowDenyPattern, node_type_pattern: AllowDenyPattern,
report: SourceReport, report: SourceReport,
node_name_pattern: AllowDenyPattern,
) -> Tuple[List[DBTNode], Optional[str], Optional[str], Optional[str], Optional[str]]: ) -> Tuple[List[DBTNode], Optional[str], Optional[str], Optional[str], Optional[str]]:
with open(manifest_path, "r") as manifest: with open(manifest_path, "r") as manifest:
dbt_manifest_json = json.load(manifest) dbt_manifest_json = json.load(manifest)
@ -307,16 +315,20 @@ def loadManifestAndCatalog(
environment, environment,
node_type_pattern, node_type_pattern,
report, report,
node_name_pattern,
) )
return nodes, manifest_schema, manifest_version, catalog_schema, catalog_version return nodes, manifest_schema, manifest_version, catalog_schema, catalog_version
def get_db_fqn(database: str, schema: str, name: str) -> str:
return f"{database}.{schema}.{name}".replace('"', "")
def get_urn_from_dbtNode( def get_urn_from_dbtNode(
database: str, schema: str, name: str, target_platform: str, env: str database: str, schema: str, name: str, target_platform: str, env: str
) -> str: ) -> str:
db_fqn = get_db_fqn(database, schema, name)
db_fqn = f"{database}.{schema}.{name}".replace('"', "")
return f"urn:li:dataset:(urn:li:dataPlatform:{target_platform},{db_fqn},{env})" return f"urn:li:dataset:(urn:li:dataPlatform:{target_platform},{db_fqn},{env})"
@ -351,7 +363,6 @@ def get_upstreams(
upstream_urns = [] upstream_urns = []
for upstream in upstreams: for upstream in upstreams:
if "identifier" in all_nodes[upstream] and use_identifiers: if "identifier" in all_nodes[upstream] and use_identifiers:
name = all_nodes[upstream]["identifier"] name = all_nodes[upstream]["identifier"]
else: else:
@ -517,6 +528,7 @@ class DBTSource(Source):
self.config.env, self.config.env,
self.config.node_type_pattern, self.config.node_type_pattern,
self.report, self.report,
self.config.node_name_pattern,
) )
additional_custom_props = { additional_custom_props = {

13
metadata-ingestion/tests/integration/dbt/dbt_manifest.json
View File

@ -2998,7 +2998,10 @@
"quoting": {}, "quoting": {},
"schema": null, "schema": null,
"tags": [], "tags": [],
"vars": {} "vars": {},
"meta": {
"owner": "@alice2"
}
}, },
"database": "pagila", "database": "pagila",
"deferred": false, "deferred": false,
@ -3021,9 +3024,7 @@
"transform", "transform",
"customer_details" "customer_details"
], ],
"meta": { "meta": {},
"owner": "@alice"
},
"name": "customer_details", "name": "customer_details",
"original_file_path": "models/transform/customer_details.sql", "original_file_path": "models/transform/customer_details.sql",
"package_name": "sample_dbt", "package_name": "sample_dbt",
@ -3390,8 +3391,8 @@
"loader": "", "loader": "",
"meta": { "meta": {
"model_maturity": "in dev", "model_maturity": "in dev",
"owner": "@alice", "some_other_property": "test 1",
"some_other_property": "test 1" "owner": "@alice1"
}, },
"name": "actor", "name": "actor",
"original_file_path": "models/base.yml", "original_file_path": "models/base.yml",

7
metadata-ingestion/tests/integration/dbt/dbt_with_schemas_mces_golden.json
View File

@ -8,7 +8,6 @@
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": { "customProperties": {
"owner": "@alice",
"node_type": "model", "node_type": "model",
"materialization": "ephemeral", "materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql", "dbt_file_path": "models/transform/customer_details.sql",
@ -29,7 +28,7 @@
"com.linkedin.pegasus2avro.common.Ownership": { "com.linkedin.pegasus2avro.common.Ownership": {
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:@alice", "owner": "urn:li:corpuser:@alice2",
"type": "DATAOWNER", "type": "DATAOWNER",
"source": null "source": null
} }
@ -644,8 +643,8 @@
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": { "customProperties": {
"model_maturity": "in dev", "model_maturity": "in dev",
"owner": "@alice",
"some_other_property": "test 1", "some_other_property": "test 1",
"owner": "@alice1",
"node_type": "source", "node_type": "source",
"dbt_file_path": "models/base.yml", "dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE", "catalog_type": "BASE TABLE",
@ -664,7 +663,7 @@
"com.linkedin.pegasus2avro.common.Ownership": { "com.linkedin.pegasus2avro.common.Ownership": {
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:@alice", "owner": "urn:li:corpuser:@alice1",
"type": "DATAOWNER", "type": "DATAOWNER",
"source": null "source": null
} }

7
metadata-ingestion/tests/integration/dbt/dbt_without_schemas_mces_golden.json
View File

@ -8,7 +8,6 @@
{ {
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": { "customProperties": {
"owner": "@alice",
"node_type": "model", "node_type": "model",
"materialization": "ephemeral", "materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql", "dbt_file_path": "models/transform/customer_details.sql",
@ -29,7 +28,7 @@
"com.linkedin.pegasus2avro.common.Ownership": { "com.linkedin.pegasus2avro.common.Ownership": {
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:@alice", "owner": "urn:li:corpuser:@alice2",
"type": "DATAOWNER", "type": "DATAOWNER",
"source": null "source": null
} }
@ -313,8 +312,8 @@
"com.linkedin.pegasus2avro.dataset.DatasetProperties": { "com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": { "customProperties": {
"model_maturity": "in dev", "model_maturity": "in dev",
"owner": "@alice",
"some_other_property": "test 1", "some_other_property": "test 1",
"owner": "@alice1",
"node_type": "source", "node_type": "source",
"dbt_file_path": "models/base.yml", "dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE", "catalog_type": "BASE TABLE",
@ -333,7 +332,7 @@
"com.linkedin.pegasus2avro.common.Ownership": { "com.linkedin.pegasus2avro.common.Ownership": {
"owners": [ "owners": [
{ {
"owner": "urn:li:corpuser:@alice", "owner": "urn:li:corpuser:@alice1",
"type": "DATAOWNER", "type": "DATAOWNER",
"source": null "source": null
} }

781
metadata-ingestion/tests/integration/dbt/dbt_without_schemas_with_filter_mces_golden.json Normal file
View File

@ -0,0 +1,781 @@
[
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "model",
"materialization": "ephemeral",
"dbt_file_path": "models/transform/customer_details.sql",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": [
"dbt:test_tag"
]
}
},
{
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [
{
"owner": "urn:li:corpuser:@alice2",
"type": "DATAOWNER",
"source": null
}
],
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
}
}
},
{
"com.linkedin.pegasus2avro.common.GlobalTags": {
"tags": [
{
"tag": "urn:li:tag:dbt:test_tag"
}
]
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": [
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"type": "TRANSFORMED"
}
]
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "model",
"materialization": "table",
"dbt_file_path": "models/billing/monthly_billing_with_cust.sql",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": [
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.customer_details,PROD)",
"type": "TRANSFORMED"
}
]
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "model",
"materialization": "view",
"dbt_file_path": "models/base/payments_base.sql",
"catalog_type": "VIEW",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": [
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"type": "TRANSFORMED"
},
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)",
"type": "TRANSFORMED"
}
]
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "model",
"materialization": "table",
"dbt_file_path": "models/transform/payments_by_customer_by_month.sql",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": [
{
"auditStamp": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"type": "TRANSFORMED"
}
]
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"model_maturity": "in dev",
"some_other_property": "test 1",
"owner": "@alice1",
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": "postgres comment: Actors table \u2013 from postgres\n\ndbt model description: description for actor table from dbt",
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [
{
"owner": "urn:li:corpuser:@alice1",
"type": "DATAOWNER",
"source": null
}
],
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
}
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": "a user's address",
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": "a user's category",
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"model_maturity": "in prod",
"owner": "@bob",
"some_other_property": "test 2",
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [
{
"owner": "urn:li:corpuser:@bob",
"type": "DATAOWNER",
"source": null
}
],
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
}
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": "description for customer table from dbt",
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"an_array_property": "['alpha', 'beta', 'charlie']",
"model_maturity": "in prod",
"owner": "@charles",
"some_other_property": "test 3",
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.common.Ownership": {
"owners": [
{
"owner": "urn:li:corpuser:@charles",
"type": "DATAOWNER",
"source": null
}
],
"lastModified": {
"time": 0,
"actor": "urn:li:corpuser:unknown",
"impersonator": null
}
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": null,
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
},
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
"urn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)",
"aspects": [
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"node_type": "source",
"dbt_file_path": "models/base.yml",
"catalog_type": "BASE TABLE",
"manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v1.json",
"manifest_version": "0.19.1",
"catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json",
"catalog_version": "0.19.1"
},
"externalUrl": null,
"description": "a payment",
"uri": null,
"tags": []
}
},
{
"com.linkedin.pegasus2avro.dataset.UpstreamLineage": {
"upstreams": []
}
}
]
}
},
"proposedDelta": null,
"systemMetadata": {
"lastObserved": 1615443388097,
"runId": "dbt-test-without-schemas-with-filter",
"properties": null
}
}
]

13
metadata-ingestion/tests/integration/dbt/test_dbt.py
View File

@ -75,6 +75,19 @@ def test_dbt_ingest(pytestconfig, tmp_path, mock_time):
"dbt_without_schemas_mces_golden.json", "dbt_without_schemas_mces_golden.json",
source_config_modifiers={"load_schemas": False}, source_config_modifiers={"load_schemas": False},
), ),
DbtTestConfig(
"dbt-test-without-schemas-with-filter",
test_resources_dir,
tmp_path,
"dbt_without_schemas_with_filter_mces.json",
"dbt_without_schemas_with_filter_mces_golden.json",
source_config_modifiers={
"load_schemas": False,
"node_name_pattern": {
"deny": ["source.sample_dbt.pagila.payment_p2020_06"]
},
},
),
] ]
for config in config_variants: for config in config_variants: