yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-12 11:17:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): conditionally include env in assertion guid (#6811)

Browse Source
This commit is contained in:
Harshal Sheth 2022-12-28 14:35:20 -05:00 committed by GitHub
parent d8af0b43bd
commit b474315e07
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 15 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -251,7 +251,7 @@ class DBTCommonConfig(StatefulIngestionConfigBase, LineageConfig):
) )
backcompat_skip_source_on_lineage_edge: bool = Field( backcompat_skip_source_on_lineage_edge: bool = Field(
False, False,
description="Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.", description="[deprecated] Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.",
) )
incremental_lineage: bool = Field( incremental_lineage: bool = Field(
@ -259,6 +259,11 @@ class DBTCommonConfig(StatefulIngestionConfigBase, LineageConfig):
default=False, default=False,
description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.", description="When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",
) )
include_env_in_assertion_guid: bool = Field(
default=False,
description="Prior to version 0.9.4.2, the assertion GUIDs did not include the environment. If you're using multiple dbt ingestion "
"that are only distinguished by env, then you should set this flag to True.",
)
stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field(
default=None, description="DBT Stateful Ingestion Config." default=None, description="DBT Stateful Ingestion Config."
) )
@ -684,6 +689,15 @@ class DBTSourceBase(StatefulIngestionSourceBase):
"platform": DBT_PLATFORM, "platform": DBT_PLATFORM,
"name": node.dbt_name, "name": node.dbt_name,
"instance": self.config.platform_instance, "instance": self.config.platform_instance,
**(
# Ideally we'd include the env unconditionally. However, we started out
# not including env in the guid, so we need to maintain backwards compatibility
# with existing PROD assertions.
{"env": self.config.env}
if self.config.env != mce_builder.DEFAULT_ENV
and self.config.include_env_in_assertion_guid
else {}
),
} }
) )
) )