yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-08 16:45:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest/dbt): add option to skip sources (#10077)

Browse Source
This commit is contained in:
Harshal Sheth 2024-03-21 01:10:24 -07:00 committed by GitHub
parent 87169baf96
commit 6c3834b38c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 1829 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
metadata-ingestion/docs/sources/dbt/dbt.md
View File

@ -269,3 +269,30 @@ source:
catalog_path: data_mart/target/catalog.json
# ... other configs
```
<details>
<summary>[Experimental] Reducing "composed of" sprawl with multiproject setups</summary>
When many dbt projects use a single table as a source, the "Composed Of" relationships can become very large and difficult to navigate.
To address this, we are experimenting with an alternative approach to handling multiproject setups: not including sources.
The benefit is that your entire dbt estate becomes much easier to navigate, and the borders between projects less noticeable.
The downside is that we will not pick up any documentation or meta mappings applied to dbt sources.
To enable this, set a few additional flags in your dbt source config:
```yaml
source:
type: dbt
config:
platform_instance: analytics
target_platform: postgres
manifest_path: analytics/target/manifest.json
catalog_path: analytics/target/catalog.json
# ... other configs
entities_enabled:
sources: No
skip_sources_in_lineage: true
```
</details>

66
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -246,6 +246,12 @@ class DBTCommonConfig(
DBTEntitiesEnabled(),
description="Controls for enabling / disabling metadata emission for different dbt entities (models, test definitions, test results, etc.)",
)
skip_sources_in_lineage: bool = Field(
default=False,
description="[Experimental] When enabled, dbt sources will not be included in the lineage graph. "
"Requires that `entities_enabled.sources` is set to `NO`. "
"This is mainly useful when you have multiple, interdependent dbt projects. ",
)
tag_prefix: str = Field(
default=f"{DBT_PLATFORM}:", description="Prefix added to tags during ingestion."
)
@ -397,6 +403,18 @@ class DBTCommonConfig(
return include_column_lineage
@validator("skip_sources_in_lineage")
def validate_skip_sources_in_lineage(
cls, skip_sources_in_lineage: bool, values: Dict
) -> bool:
entites_enabled: DBTEntitiesEnabled = values["entities_enabled"]
if skip_sources_in_lineage and entites_enabled.sources == EmitDirective.YES:
raise ValueError(
"When `skip_sources_in_lineage` is enabled, `entities_enabled.sources` must be set to NO."
)
return skip_sources_in_lineage
@dataclass
class DBTColumn:
@ -510,11 +528,13 @@ class DBTNode:
target_platform: str,
target_platform_instance: Optional[str],
env: str,
skip_sources_in_lineage: bool,
) -> str:
"""
Get the urn to use when referencing this node in a dbt node's upstream lineage.
If the node is a source or an ephemeral dbt node, we should point at the dbt node.
If the node is an ephemeral dbt node, we should point at the dbt node.
If the node is a source node, and skip_sources_in_lineage is not enabled, we should also point at the dbt node.
Otherwise, the node is materialized in the target platform, and so lineage should
point there.
"""
@ -523,14 +543,11 @@ class DBTNode:
platform_value = DBT_PLATFORM
platform_instance_value = dbt_platform_instance
materialized = self.materialization
if materialized in {
"view",
"materialized_view",
"table",
"incremental",
"snapshot",
}:
if self.is_ephemeral_model():
pass # leave it pointing at dbt
elif self.node_type == "source" and not skip_sources_in_lineage:
pass # leave it as dbt
else:
# upstream urns point to the target platform
platform_value = target_platform
platform_instance_value = target_platform_instance
@ -623,6 +640,7 @@ def get_upstreams(
target_platform_instance: Optional[str],
environment: str,
platform_instance: Optional[str],
skip_sources_in_lineage: bool,
) -> List[str]:
upstream_urns = []
@ -642,6 +660,7 @@ def get_upstreams(
target_platform=target_platform,
target_platform_instance=target_platform_instance,
env=environment,
skip_sources_in_lineage=skip_sources_in_lineage,
)
)
return upstream_urns
@ -818,9 +837,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
if self.config.entities_enabled.can_emit_node_type("test"):
yield MetadataChangeProposalWrapper(
entityUrn=assertion_urn,
aspect=DataPlatformInstanceClass(
platform=mce_builder.make_data_platform_urn(DBT_PLATFORM)
),
aspect=self._make_data_platform_instance_aspect(),
).as_workunit()
upstream_urns = get_upstreams_for_test(
@ -868,6 +885,17 @@ class DBTSourceBase(StatefulIngestionSourceBase):
self.stale_entity_removal_handler.workunit_processor,
]
def _make_data_platform_instance_aspect(self) -> DataPlatformInstanceClass:
return DataPlatformInstanceClass(
platform=mce_builder.make_data_platform_urn(DBT_PLATFORM),
instance=mce_builder.make_dataplatform_instance_urn(
mce_builder.make_data_platform_urn(DBT_PLATFORM),
self.config.platform_instance,
)
if self.config.platform_instance
else None,
)
def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]:
if self.config.write_semantics == "PATCH":
self.ctx.require_graph("Using dbt with write_semantics=PATCH")
@ -1190,6 +1218,12 @@ class DBTSourceBase(StatefulIngestionSourceBase):
if sub_type_wu:
yield sub_type_wu
# emit dataPlatformInstance aspect
yield MetadataChangeProposalWrapper(
entityUrn=node_datahub_urn,
aspect=self._make_data_platform_instance_aspect(),
).as_workunit()
if len(aspects) == 0:
continue
dataset_snapshot = DatasetSnapshot(
@ -1525,12 +1559,6 @@ class DBTSourceBase(StatefulIngestionSourceBase):
return None
subtypes: List[str] = [node.node_type.capitalize()]
if node.node_type == "source":
# In the siblings association hook, we previously looked for an exact
# match of "source" to determine if a node was a source. While we now
# also check for a capitalized "Source" subtype, this maintains compatibility
# with older GMS versions.
subtypes.append("source")
if node.materialization == "table":
subtypes.append(DatasetSubTypes.TABLE)
@ -1578,6 +1606,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
self.config.target_platform_instance,
self.config.env,
self.config.platform_instance,
skip_sources_in_lineage=self.config.skip_sources_in_lineage,
)
def _translate_dbt_name_to_upstream_urn(dbt_name: str) -> str:
@ -1586,6 +1615,7 @@ class DBTSourceBase(StatefulIngestionSourceBase):
target_platform=self.config.target_platform,
target_platform_instance=self.config.target_platform_instance,
env=self.config.env,
skip_sources_in_lineage=self.config.skip_sources_in_lineage,
)
if node.cll_debug_info and node.cll_debug_info.error:

292
metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json
View File

@ -18,6 +18,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -368,6 +384,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -585,6 +617,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -907,6 +955,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1070,8 +1134,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1081,6 +1144,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1289,8 +1368,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1300,6 +1378,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1569,8 +1663,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1580,6 +1673,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1734,8 +1843,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1745,6 +1853,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1922,8 +2046,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1933,6 +2056,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2108,8 +2247,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2119,6 +2257,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2434,8 +2588,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2445,6 +2598,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2668,8 +2837,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2679,6 +2847,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2924,8 +3108,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2935,6 +3118,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3158,8 +3357,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3169,6 +3367,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3392,8 +3606,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3403,6 +3616,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3626,8 +3855,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3637,6 +3865,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-schemas-dbt-enabled",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

308
metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json
View File

@ -18,6 +18,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -333,6 +349,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -538,6 +570,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -852,6 +900,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1054,6 +1118,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1426,8 +1506,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1437,6 +1516,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1639,8 +1734,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1650,6 +1744,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1919,8 +2029,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1930,6 +2039,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2084,8 +2209,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2095,6 +2219,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2272,8 +2412,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2283,6 +2422,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2458,8 +2613,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2469,6 +2623,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2784,8 +2954,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2795,6 +2964,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3018,8 +3203,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3029,6 +3213,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3274,8 +3474,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3285,6 +3484,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3508,8 +3723,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3519,6 +3733,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3742,8 +3972,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3753,6 +3982,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3976,8 +4221,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3987,6 +4231,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-column-meta-mapping",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

134
metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json
View File

@ -19,6 +19,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.customers,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -229,6 +245,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.orders,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -457,6 +489,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_customers,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -554,7 +602,7 @@
"time": 1643871600000,
"actor": "urn:li:corpuser:unknown"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)",
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)",
"type": "TRANSFORMED"
}
]
@ -596,6 +644,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_orders,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -705,7 +769,7 @@
"time": 1643871600000,
"actor": "urn:li:corpuser:unknown"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)",
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)",
"type": "TRANSFORMED"
}
]
@ -747,6 +811,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.stg_payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -856,7 +936,7 @@
"time": 1643871600000,
"actor": "urn:li:corpuser:unknown"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)",
"dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)",
"type": "TRANSFORMED"
}
]
@ -897,6 +977,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_customers,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1013,6 +1109,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_orders,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1141,6 +1253,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,calm-pagoda-323403.jaffle_shop.raw_payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-2022_02_03-07_00_00",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

273
metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json
View File

@ -18,6 +18,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -332,6 +348,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -531,6 +563,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -853,6 +901,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1016,8 +1080,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1027,6 +1090,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1232,8 +1311,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1243,6 +1321,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1512,8 +1606,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1523,6 +1616,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1677,8 +1786,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1688,6 +1796,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1865,8 +1989,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1876,6 +1999,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2048,8 +2187,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2059,6 +2197,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2374,8 +2528,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2385,6 +2538,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2608,8 +2777,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2619,6 +2787,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2861,8 +3045,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2872,6 +3055,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3095,8 +3294,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3106,6 +3304,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3329,8 +3543,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3340,6 +3553,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-complex-owner-patterns",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

308
metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json
View File

@ -18,6 +18,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.customer_details,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -333,6 +350,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -532,6 +566,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -854,6 +905,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1017,8 +1085,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1028,6 +1095,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.actor,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1233,8 +1317,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1244,6 +1327,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1513,8 +1613,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1524,6 +1623,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.category,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1678,8 +1794,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1689,6 +1804,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1866,8 +1998,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1877,6 +2008,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.country,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2049,8 +2197,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2060,6 +2207,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2375,8 +2539,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2386,6 +2549,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_01,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2609,8 +2789,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2620,6 +2799,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2862,8 +3058,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2873,6 +3068,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_03,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3096,8 +3308,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3107,6 +3318,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_04,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3330,8 +3558,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3341,6 +3568,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3564,8 +3808,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3575,6 +3818,23 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt",
"instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dbt,dbt-instance-1)"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-data-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

292
metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json
View File

@ -18,6 +18,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -333,6 +349,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -532,6 +564,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -854,6 +902,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1017,8 +1081,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1028,6 +1091,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1233,8 +1312,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1244,6 +1322,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1513,8 +1607,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1524,6 +1617,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1678,8 +1787,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1689,6 +1797,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1866,8 +1990,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1877,6 +2000,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2049,8 +2188,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2060,6 +2198,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2375,8 +2529,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2386,6 +2539,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2609,8 +2778,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2620,6 +2788,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2862,8 +3046,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2873,6 +3056,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3096,8 +3295,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3107,6 +3305,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3330,8 +3544,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3341,6 +3554,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3564,8 +3793,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3575,6 +3803,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-non-incremental-lineage",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {

292
metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json
View File

@ -18,6 +18,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.customer_details,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -333,6 +349,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-monthly-billing,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -532,6 +564,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.an-aliased-view-for-payments,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -854,6 +902,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.dbt_postgres.payments_by_customer_by_month,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1017,8 +1081,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1028,6 +1091,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1233,8 +1312,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1244,6 +1322,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1513,8 +1607,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1524,6 +1617,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1678,8 +1787,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1689,6 +1797,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -1866,8 +1990,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -1877,6 +2000,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2049,8 +2188,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2060,6 +2198,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2375,8 +2529,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2386,6 +2539,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2609,8 +2778,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2620,6 +2788,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -2862,8 +3046,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -2873,6 +3056,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3096,8 +3295,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3107,6 +3305,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3330,8 +3544,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3341,6 +3554,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {
@ -3564,8 +3793,7 @@
"aspect": {
"json": {
"typeNames": [
"Source",
"source"
"Source"
]
}
},
@ -3575,6 +3803,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:dbt"
}
},
"systemMetadata": {
"lastObserved": 1643871600000,
"runId": "dbt-test-with-target-platform-instance",
"lastRunId": "no-run-id-provided"
}
},
{
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": {