From b474315e07025f73d17de5bc06385d5a3b13ece2 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 28 Dec 2022 14:35:20 -0500 Subject: [PATCH] fix(ingest): conditionally include env in assertion guid (#6811) --- .../datahub/ingestion/source/dbt/dbt_common.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 44338c3d7d..adfbfe1c8d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -251,7 +251,7 @@ class DBTCommonConfig(StatefulIngestionConfigBase, LineageConfig): ) backcompat_skip_source_on_lineage_edge: bool = Field( False, - description="Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.", + description="[deprecated] Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.", ) incremental_lineage: bool = Field( @@ -259,6 +259,11 @@ class DBTCommonConfig(StatefulIngestionConfigBase, LineageConfig): default=False, description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", ) + include_env_in_assertion_guid: bool = Field( + default=False, + description="Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion " + "that are only distinguished by env, then you should set this flag to True.", + ) stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( default=None, description="DBT Stateful Ingestion Config." ) @@ -684,6 +689,15 @@ class DBTSourceBase(StatefulIngestionSourceBase): "platform": DBT_PLATFORM, "name": node.dbt_name, "instance": self.config.platform_instance, + **( + # Ideally we'd include the env unconditionally. However, we started out + # not including env in the guid, so we need to maintain backwards compatibility + # with existing PROD assertions. + {"env": self.config.env} + if self.config.env != mce_builder.DEFAULT_ENV + and self.config.include_env_in_assertion_guid + else {} + ), } ) )