diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt.py b/metadata-ingestion/src/datahub/ingestion/source/dbt.py index b5889c5bc0..f3ab2a8a31 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt.py @@ -216,6 +216,10 @@ class DBTConfig(StatefulIngestionConfigBase): False, description="Prior to version 0.8.38, dbt tests were represented as datasets. If you ingested dbt tests before, set this flag to True (just needed once) to soft-delete tests that were generated as datasets by previous ingestion.", ) + backcompat_skip_source_on_lineage_edge: bool = Field( + False, + description="Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.", + ) @property def s3_client(self): @@ -544,6 +548,7 @@ def get_upstreams( environment: str, disable_dbt_node_creation: bool, platform_instance: Optional[str], + legacy_skip_source_lineage: Optional[bool], ) -> List[str]: upstream_urns = [] @@ -579,9 +584,8 @@ def get_upstreams( materialized = upstream_manifest_node.get("config", {}).get("materialized") resource_type = upstream_manifest_node["resource_type"] - if ( - materialized in {"view", "table", "incremental"} - or resource_type == "source" + if materialized in {"view", "table", "incremental"} or ( + resource_type == "source" and legacy_skip_source_lineage ): # upstream urns point to the target platform platform_value = target_platform @@ -873,6 +877,7 @@ class DBTTest: config.env, config.disable_dbt_node_creation, config.platform_instance, + config.backcompat_skip_source_on_lineage_edge, ) assertion_urn = mce_builder.make_assertion_urn( mce_builder.datahub_guid( @@ -1161,6 +1166,7 @@ class DBTSource(StatefulIngestionSourceBase): environment=self.config.env, disable_dbt_node_creation=self.config.disable_dbt_node_creation, platform_instance=None, + legacy_skip_source_lineage=self.config.backcompat_skip_source_on_lineage_edge, ) raw_node = manifest_nodes.get(node.dbt_name) @@ -1756,6 +1762,7 @@ class DBTSource(StatefulIngestionSourceBase): self.config.env, self.config.disable_dbt_node_creation, self.config.platform_instance, + self.config.backcompat_skip_source_on_lineage_edge, ) # if a node is of type source in dbt, its upstream lineage should have the corresponding table/view @@ -1791,6 +1798,7 @@ class DBTSource(StatefulIngestionSourceBase): self.config.env, self.config.disable_dbt_node_creation, self.config.platform_instance, + self.config.backcompat_skip_source_on_lineage_edge, ) if upstream_urns: return get_upstream_lineage(upstream_urns) diff --git a/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json index da6389769a..3ca9cc2b96 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_deleted_actor_mces_golden.json @@ -79,6 +79,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -89,6 +91,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -96,6 +99,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -106,6 +111,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -113,6 +119,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -123,6 +131,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -254,7 +263,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -263,7 +272,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -272,7 +281,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -378,6 +387,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -388,6 +399,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -395,6 +407,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -405,6 +419,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -412,6 +427,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -422,6 +439,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -429,6 +447,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -439,6 +459,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -571,6 +592,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -581,6 +604,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -588,6 +612,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -598,6 +624,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -605,6 +632,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -615,6 +644,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -622,6 +652,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -632,6 +664,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -639,6 +672,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -649,6 +684,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -656,6 +692,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -666,6 +704,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -683,7 +722,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -692,7 +731,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" }, { @@ -701,7 +740,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -710,7 +749,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -719,7 +758,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -728,7 +767,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" } ], @@ -833,6 +872,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -843,6 +884,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -850,6 +892,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -860,6 +904,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -867,6 +912,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -877,6 +924,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -884,6 +932,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -894,6 +944,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -901,6 +952,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -911,6 +964,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -918,6 +972,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -928,6 +984,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -935,6 +992,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -945,6 +1004,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -952,6 +1012,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -962,6 +1024,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1077,6 +1140,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1087,6 +1152,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1094,6 +1160,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1104,6 +1172,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1111,6 +1180,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1121,6 +1192,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1236,6 +1308,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1246,6 +1320,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1253,6 +1328,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1263,6 +1340,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1270,6 +1348,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1280,6 +1360,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1287,6 +1368,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1297,6 +1380,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1431,6 +1515,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1441,6 +1527,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1448,6 +1535,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1458,6 +1547,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1465,6 +1555,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1475,6 +1567,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1590,6 +1683,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1600,6 +1695,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1607,6 +1703,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1617,6 +1715,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1624,6 +1723,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1634,6 +1735,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1641,6 +1743,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1651,6 +1755,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1658,6 +1763,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1668,6 +1775,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1675,6 +1783,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1685,6 +1795,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1692,6 +1803,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1702,6 +1815,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1709,6 +1823,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1719,6 +1835,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1726,6 +1843,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1736,6 +1855,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1743,6 +1863,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1753,6 +1875,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1868,6 +1991,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1878,6 +2003,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1885,6 +2011,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1895,6 +2023,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1902,6 +2031,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1912,6 +2043,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1919,6 +2051,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1929,6 +2063,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1936,6 +2071,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1946,6 +2083,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1953,6 +2091,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1963,6 +2103,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2098,6 +2239,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2108,6 +2251,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2115,6 +2259,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2271,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2132,6 +2279,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2142,6 +2291,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2149,6 +2299,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2159,6 +2311,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2166,6 +2319,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2176,6 +2331,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2183,6 +2339,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2193,6 +2351,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2308,6 +2467,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2318,6 +2479,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2325,6 +2487,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2335,6 +2499,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2342,6 +2507,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2352,6 +2519,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2359,6 +2527,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2369,6 +2539,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2376,6 +2547,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2386,6 +2559,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2393,6 +2567,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2403,6 +2579,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2518,6 +2695,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2528,6 +2707,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2535,6 +2715,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2545,6 +2727,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2552,6 +2735,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2562,6 +2747,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2569,6 +2755,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2579,6 +2767,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2586,6 +2775,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2596,6 +2787,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2603,6 +2795,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2613,6 +2807,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2728,6 +2923,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2738,6 +2935,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2745,6 +2943,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2755,6 +2955,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2762,6 +2963,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2772,6 +2975,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2779,6 +2983,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2789,6 +2995,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2796,6 +3003,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2806,6 +3015,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2813,6 +3023,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2823,6 +3035,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2938,6 +3151,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2948,6 +3163,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2955,6 +3171,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2965,6 +3183,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2972,6 +3191,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2982,6 +3203,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2989,6 +3211,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2999,6 +3223,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3006,6 +3231,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3016,6 +3243,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3023,6 +3251,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3033,6 +3263,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -3174,7 +3405,7 @@ { "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", @@ -3193,7 +3424,7 @@ { "auditHeader": null, "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "status", diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index d7b8d0c245..22364c7dee 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -163,7 +163,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -172,7 +172,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -181,7 +181,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -646,7 +646,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -655,7 +655,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -664,7 +664,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -673,7 +673,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -682,7 +682,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -691,7 +691,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -700,7 +700,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json index 15bb69ca9a..16c1ea7b1c 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_mces_golden.json @@ -121,7 +121,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -130,7 +130,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -139,7 +139,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -585,7 +585,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -594,7 +594,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -603,7 +603,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -612,7 +612,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -621,7 +621,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -630,7 +630,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -639,7 +639,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json index 43a4f9b7d9..52c2a1c614 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_without_schemas_with_filter_mces_golden.json @@ -121,7 +121,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -130,7 +130,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -139,7 +139,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -585,7 +585,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -594,7 +594,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -603,7 +603,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -612,7 +612,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -621,7 +621,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -630,7 +630,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -639,7 +639,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index c8d94f8ff3..6bc814bdf8 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -75,7 +75,8 @@ "com.linkedin.pegasus2avro.common.GlobalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:test_tag" + "tag": "urn:li:tag:dbt:test_tag", + "context": null } ] } @@ -119,7 +120,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -128,7 +129,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -137,7 +138,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -248,6 +249,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -258,6 +261,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -265,6 +269,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -275,6 +281,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -282,6 +289,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -292,6 +301,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -299,6 +309,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -309,6 +321,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -441,6 +454,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -451,6 +466,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -458,6 +474,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -468,6 +486,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -475,6 +494,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -485,6 +506,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -492,6 +514,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -502,6 +526,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -509,6 +534,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -519,6 +546,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -526,6 +554,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -536,6 +566,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -553,7 +584,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -562,7 +593,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -571,7 +602,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -580,7 +611,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -589,7 +620,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -598,7 +629,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -607,7 +638,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], @@ -713,6 +744,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -723,6 +756,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -730,6 +764,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -740,6 +776,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -747,6 +784,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -757,6 +796,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -898,6 +938,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -908,6 +950,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -915,6 +958,8 @@ "jsonPath": null, "nullable": false, "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -925,12 +970,14 @@ "globalTags": { "tags": [ { - "tag": "urn:li:tag:dbt:column_tag" + "tag": "urn:li:tag:dbt:column_tag", + "context": null } ] }, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -938,6 +985,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_name from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -948,6 +997,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -955,6 +1005,8 @@ "jsonPath": null, "nullable": false, "description": "description for last_update from dbt", + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -965,6 +1017,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1080,6 +1133,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1090,6 +1145,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1097,6 +1153,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1107,6 +1165,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1114,6 +1173,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1124,6 +1185,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1131,6 +1193,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1141,6 +1205,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1148,6 +1213,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1158,6 +1225,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1165,6 +1233,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1175,6 +1245,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1182,6 +1253,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1192,6 +1265,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1199,6 +1273,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1209,6 +1285,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1324,6 +1401,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1334,6 +1413,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1341,6 +1421,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1351,6 +1433,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1358,6 +1441,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1368,6 +1453,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1483,6 +1569,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1493,6 +1581,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1500,6 +1589,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1510,6 +1601,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1517,6 +1609,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1527,6 +1621,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1534,6 +1629,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1544,6 +1641,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1678,6 +1776,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1688,6 +1788,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1695,6 +1796,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1705,6 +1808,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1712,6 +1816,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1722,6 +1828,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -1837,6 +1944,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1847,6 +1956,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1854,6 +1964,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.BooleanType": {} @@ -1864,6 +1976,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1871,6 +1984,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1881,6 +1996,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1888,6 +2004,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.DateType": {} @@ -1898,6 +2016,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1905,6 +2024,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -1915,6 +2036,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1922,6 +2044,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1932,6 +2056,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1939,6 +2064,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1949,6 +2076,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1956,6 +2084,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.StringType": {} @@ -1966,6 +2096,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1973,6 +2104,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -1983,6 +2116,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -1990,6 +2124,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2000,6 +2136,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2115,6 +2252,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2125,6 +2264,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2132,6 +2272,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2142,6 +2284,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2149,6 +2292,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2159,6 +2304,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2166,6 +2312,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2176,6 +2324,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2183,6 +2332,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2193,6 +2344,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2200,6 +2352,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2210,6 +2364,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2345,6 +2500,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2355,6 +2512,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2362,6 +2520,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2372,6 +2532,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2379,6 +2540,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2389,6 +2552,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2396,6 +2560,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2406,6 +2572,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2413,6 +2580,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2423,6 +2592,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2430,6 +2600,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2440,6 +2612,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2555,6 +2728,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2565,6 +2740,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2572,6 +2748,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2582,6 +2760,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2589,6 +2768,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2599,6 +2780,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2606,6 +2788,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2616,6 +2800,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2623,6 +2808,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2633,6 +2820,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2640,6 +2828,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2650,6 +2840,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2765,6 +2956,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2775,6 +2968,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2782,6 +2976,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2792,6 +2988,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2799,6 +2996,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -2809,6 +3008,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2816,6 +3016,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2826,6 +3028,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2833,6 +3036,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2843,6 +3048,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2850,6 +3056,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2860,6 +3068,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], @@ -2975,6 +3184,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -2985,6 +3196,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -2992,6 +3204,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3002,6 +3216,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3009,6 +3224,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.TimeType": {} @@ -3019,6 +3236,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3026,6 +3244,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3036,6 +3256,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3043,6 +3264,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3053,6 +3276,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null }, { @@ -3060,6 +3284,8 @@ "jsonPath": null, "nullable": false, "description": null, + "created": null, + "lastModified": null, "type": { "type": { "com.linkedin.pegasus2avro.schema.NumberType": {} @@ -3070,6 +3296,7 @@ "globalTags": null, "glossaryTerms": null, "isPartOfKey": false, + "isPartitioningKey": null, "jsonProps": null } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index f9841f806a..18ae27de4d 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -121,7 +121,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -130,7 +130,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -139,7 +139,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -585,7 +585,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -594,7 +594,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -603,7 +603,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -612,7 +612,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -621,7 +621,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -630,7 +630,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -639,7 +639,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,dbt-instance-1.pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ], diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index f72b75cd00..452dc14cd1 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -121,7 +121,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.customer,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", "type": "TRANSFORMED" }, { @@ -130,7 +130,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.address,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", "type": "TRANSFORMED" }, { @@ -139,7 +139,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.city,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", "type": "TRANSFORMED" } ], @@ -585,7 +585,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_01,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", "type": "TRANSFORMED" }, { @@ -594,7 +594,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -603,7 +603,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_02,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", "type": "TRANSFORMED" }, { @@ -612,7 +612,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_03,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", "type": "TRANSFORMED" }, { @@ -621,7 +621,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_04,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", "type": "TRANSFORMED" }, { @@ -630,7 +630,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_05,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", "type": "TRANSFORMED" }, { @@ -639,7 +639,7 @@ "actor": "urn:li:corpuser:unknown", "impersonator": null }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,ps-instance-1.pagila.public.payment_p2020_06,PROD)", + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", "type": "TRANSFORMED" } ],