yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-16 12:38:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/powerbi): fix broken lineage between chart and dataset (#11080)

Browse Source
This commit is contained in:
sid-acryl 2024-08-07 09:32:21 +05:30 committed by GitHub
parent 543e447787
commit 2755cf3559
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 833 additions and 816 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
View File

@ -89,6 +89,7 @@ from datahub.metadata.schema_classes import (
from datahub.metadata.urns import ChartUrn from datahub.metadata.urns import ChartUrn
from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo from datahub.sql_parsing.sqlglot_lineage import ColumnLineageInfo
from datahub.utilities.dedup_list import deduplicate_list from datahub.utilities.dedup_list import deduplicate_list
from datahub.utilities.urns.urn_iter import lowercase_dataset_urn
# Logger instance # Logger instance
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -127,7 +128,7 @@ class Mapper:
@staticmethod @staticmethod
def urn_to_lowercase(value: str, flag: bool) -> str: def urn_to_lowercase(value: str, flag: bool) -> str:
if flag is True: if flag is True:
return value.lower() return lowercase_dataset_urn(value)
return value return value
@ -390,11 +391,13 @@ class Mapper:
for table in dataset.tables: for table in dataset.tables:
# Create a URN for dataset # Create a URN for dataset
ds_urn = builder.make_dataset_urn_with_platform_instance( ds_urn = self.assets_urn_to_lowercase(
platform=self.__config.platform_name, builder.make_dataset_urn_with_platform_instance(
name=self.assets_urn_to_lowercase(table.full_name), platform=self.__config.platform_name,
platform_instance=self.__config.platform_instance, name=table.full_name,
env=self.__config.env, platform_instance=self.__config.platform_instance,
env=self.__config.env,
)
) )
logger.debug(f"dataset_urn={ds_urn}") logger.debug(f"dataset_urn={ds_urn}")

6
metadata-ingestion/src/datahub/utilities/urns/urn_iter.py
View File

@ -131,7 +131,7 @@ def _modify_at_path(
_modify_at_path(getattr(model, path[0]), path[1:], new_value) _modify_at_path(getattr(model, path[0]), path[1:], new_value)
def _lowercase_dataset_urn(dataset_urn: str) -> str: def lowercase_dataset_urn(dataset_urn: str) -> str:
cur_urn = DatasetUrn.from_string(dataset_urn) cur_urn = DatasetUrn.from_string(dataset_urn)
new_urn = DatasetUrn( new_urn = DatasetUrn(
platform=cur_urn.platform, name=cur_urn.name.lower(), env=cur_urn.env platform=cur_urn.platform, name=cur_urn.name.lower(), env=cur_urn.env
@ -149,10 +149,10 @@ def lowercase_dataset_urns(
) -> None: ) -> None:
def modify_urn(urn: str) -> str: def modify_urn(urn: str) -> str:
if guess_entity_type(urn) == "dataset": if guess_entity_type(urn) == "dataset":
return _lowercase_dataset_urn(urn) return lowercase_dataset_urn(urn)
elif guess_entity_type(urn) == "schemaField": elif guess_entity_type(urn) == "schemaField":
cur_urn = Urn.from_string(urn) cur_urn = Urn.from_string(urn)
cur_urn._entity_ids[0] = _lowercase_dataset_urn(cur_urn._entity_ids[0]) cur_urn._entity_ids[0] = lowercase_dataset_urn(cur_urn._entity_ids[0])
return str(cur_urn) return str(cur_urn)
return urn return urn

1626
metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
View File

File diff suppressed because it is too large Load Diff

2
metadata-ingestion/tests/integration/powerbi/test_powerbi.py
View File

@ -819,6 +819,8 @@ def test_powerbi_ingest_urn_lower_case(
"type": "powerbi", "type": "powerbi",
"config": { "config": {
**default_source_config(), **default_source_config(),
"env": "PROD",
"platform_instance": "myPlatformInstance",
"convert_urns_to_lowercase": True, "convert_urns_to_lowercase": True,
"convert_lineage_urns_to_lowercase": True, "convert_lineage_urns_to_lowercase": True,
}, },