yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-19 14:44:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(dbt): support prefer_sql_parser_lineage with sources enabled (#11168)

Browse Source
This commit is contained in:
Harshal Sheth 2024-08-13 13:54:50 -07:00 committed by GitHub
parent d36edcace9
commit 897173f270
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 3022 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py
View File

@ -276,6 +276,12 @@ class DBTCommonConfig(
DBTEntitiesEnabled(), DBTEntitiesEnabled(),
description="Controls for enabling / disabling metadata emission for different dbt entities (models, test definitions, test results, etc.)", description="Controls for enabling / disabling metadata emission for different dbt entities (models, test definitions, test results, etc.)",
) )
prefer_sql_parser_lineage: bool = Field(
default=False,
description="Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. "
"This can be useful when dbt models reference tables directly, instead of using the ref() macro. "
"This requires that `skip_sources_in_lineage` is enabled.",
)
skip_sources_in_lineage: bool = Field( skip_sources_in_lineage: bool = Field(
default=False, default=False,
description="[Experimental] When enabled, dbt sources will not be included in the lineage graph. " description="[Experimental] When enabled, dbt sources will not be included in the lineage graph. "
@ -366,13 +372,6 @@ class DBTCommonConfig(
description="When enabled, includes the compiled code in the emitted metadata.", description="When enabled, includes the compiled code in the emitted metadata.",
) )
prefer_sql_parser_lineage: bool = Field(
default=False,
description="Normally we use dbt's metadata to generate table lineage. When enabled, we prefer results from the SQL parser when generating lineage instead. "
"This can be useful when dbt models reference tables directly, instead of using the ref() macro. "
"This requires that `skip_sources_in_lineage` is enabled.",
)
@validator("target_platform") @validator("target_platform")
def validate_target_platform_value(cls, target_platform: str) -> str: def validate_target_platform_value(cls, target_platform: str) -> str:
if target_platform.lower() == DBT_PLATFORM: if target_platform.lower() == DBT_PLATFORM:
@ -438,15 +437,27 @@ class DBTCommonConfig(
return include_column_lineage return include_column_lineage
@validator("skip_sources_in_lineage") @validator("skip_sources_in_lineage", always=True)
def validate_skip_sources_in_lineage( def validate_skip_sources_in_lineage(
cls, skip_sources_in_lineage: bool, values: Dict cls, skip_sources_in_lineage: bool, values: Dict
) -> bool: ) -> bool:
entites_enabled: Optional[DBTEntitiesEnabled] = values.get("entities_enabled") entities_enabled: Optional[DBTEntitiesEnabled] = values.get("entities_enabled")
prefer_sql_parser_lineage: Optional[bool] = values.get(
"prefer_sql_parser_lineage"
)
if prefer_sql_parser_lineage and not skip_sources_in_lineage:
raise ValueError(
"`prefer_sql_parser_lineage` requires that `skip_sources_in_lineage` is enabled."
)
if ( if (
skip_sources_in_lineage skip_sources_in_lineage
and entites_enabled and entities_enabled
and entites_enabled.sources == EmitDirective.YES and entities_enabled.sources == EmitDirective.YES
# When `prefer_sql_parser_lineage` is enabled, it's ok to have `skip_sources_in_lineage` enabled
# without also disabling sources.
and not prefer_sql_parser_lineage
): ):
raise ValueError( raise ValueError(
"When `skip_sources_in_lineage` is enabled, `entities_enabled.sources` must be set to NO." "When `skip_sources_in_lineage` is enabled, `entities_enabled.sources` must be set to NO."
@ -454,16 +465,6 @@ class DBTCommonConfig(
return skip_sources_in_lineage return skip_sources_in_lineage
@validator("prefer_sql_parser_lineage")
def validate_prefer_sql_parser_lineage(
cls, prefer_sql_parser_lineage: bool, values: Dict
) -> bool:
if prefer_sql_parser_lineage and not values.get("skip_sources_in_lineage"):
raise ValueError(
"`prefer_sql_parser_lineage` requires that `skip_sources_in_lineage` is enabled."
)
return prefer_sql_parser_lineage
@dataclass @dataclass
class DBTColumn: class DBTColumn:

3031
metadata-ingestion/tests/integration/dbt/dbt_test_prefer_sql_parser_lineage_golden.json
View File

File diff suppressed because it is too large Load Diff

2
metadata-ingestion/tests/integration/dbt/test_dbt.py
View File

@ -227,7 +227,7 @@ class DbtTestConfig:
source_config_modifiers={ source_config_modifiers={
"prefer_sql_parser_lineage": True, "prefer_sql_parser_lineage": True,
"skip_sources_in_lineage": True, "skip_sources_in_lineage": True,
"entities_enabled": {"sources": "NO"}, # "entities_enabled": {"sources": "NO"},
}, },
), ),
], ],

1
metadata-ingestion/tests/unit/test_dbt_source.py
View File

@ -247,7 +247,6 @@ def test_dbt_config_prefer_sql_parser_lineage():
"catalog_path": "dummy_path", "catalog_path": "dummy_path",
"target_platform": "dummy_platform", "target_platform": "dummy_platform",
"skip_sources_in_lineage": True, "skip_sources_in_lineage": True,
"entities_enabled": {"sources": "NO"},
"prefer_sql_parser_lineage": True, "prefer_sql_parser_lineage": True,
} }
config = DBTCoreConfig.parse_obj(config_dict) config = DBTCoreConfig.parse_obj(config_dict)