yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-19 06:03:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/dbt): remove deprecated backcompat_skip_source_on_lineage_edge option (#7466)

Browse Source
This commit is contained in:
Harshal Sheth 2023-03-02 02:24:50 -05:00 committed by GitHub
parent 8101f0d47a
commit 619fad0ae1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -260,13 +260,6 @@ class DBTCommonConfig(StatefulIngestionConfigBase, LineageConfig):
default=None, default=None,
description='Regex string to extract owner from the dbt node using the `(?P<name>...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P<owner>(.*)): (\\w+) (\\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P<owner>(.*))"` will extract `alice` as the owner from `"@alice"`.', description='Regex string to extract owner from the dbt node using the `(?P<name>...) syntax` of the [match object](https://docs.python.org/3/library/re.html#match-objects), where the group name must be `owner`. Examples: (1)`r"(?P<owner>(.*)): (\\w+) (\\w+)"` will extract `jdoe` as the owner from `"jdoe: John Doe"` (2) `r"@(?P<owner>(.*))"` will extract `alice` as the owner from `"@alice"`.',
) )
backcompat_skip_source_on_lineage_edge: bool = Field(
False,
description="[deprecated] Prior to version 0.8.41, lineage edges to sources were directed to the target platform node rather than the dbt source node. This contradicted the established pattern for other lineage edges to point to upstream dbt nodes. To revert lineage logic to this legacy approach, set this flag to true.",
)
_deprecate_skip_source_on_lineage_edge = pydantic_field_deprecated(
"backcompat_skip_source_on_lineage_edge"
)
incremental_lineage: bool = Field( incremental_lineage: bool = Field(
# Copied from LineageConfig, and changed the default. # Copied from LineageConfig, and changed the default.
@ -438,7 +431,6 @@ def get_upstreams(
target_platform_instance: Optional[str], target_platform_instance: Optional[str],
environment: str, environment: str,
platform_instance: Optional[str], platform_instance: Optional[str],
legacy_skip_source_lineage: Optional[bool],
) -> List[str]: ) -> List[str]:
upstream_urns = [] upstream_urns = []
@ -457,10 +449,7 @@ def get_upstreams(
materialized = upstream_manifest_node.materialization materialized = upstream_manifest_node.materialization
resource_type = upstream_manifest_node.node_type if materialized in {"view", "table", "incremental", "snapshot"}:
if materialized in {"view", "table", "incremental", "snapshot"} or (
resource_type == "source" and legacy_skip_source_lineage
):
# upstream urns point to the target platform # upstream urns point to the target platform
platform_value = target_platform platform_value = target_platform
platform_instance_value = target_platform_instance platform_instance_value = target_platform_instance
@ -732,7 +721,6 @@ class DBTSourceBase(StatefulIngestionSourceBase):
target_platform_instance=self.config.target_platform_instance, target_platform_instance=self.config.target_platform_instance,
environment=self.config.env, environment=self.config.env,
platform_instance=None, platform_instance=None,
legacy_skip_source_lineage=self.config.backcompat_skip_source_on_lineage_edge,
) )
for upstream_urn in sorted(upstream_urns): for upstream_urn in sorted(upstream_urns):
@ -1367,7 +1355,6 @@ class DBTSourceBase(StatefulIngestionSourceBase):
self.config.target_platform_instance, self.config.target_platform_instance,
self.config.env, self.config.env,
self.config.platform_instance, self.config.platform_instance,
self.config.backcompat_skip_source_on_lineage_edge,
) )
# if a node is of type source in dbt, its upstream lineage should have the corresponding table/view # if a node is of type source in dbt, its upstream lineage should have the corresponding table/view