From cf7a442e32709e6bf0abd7d954f621fd5730e7a9 Mon Sep 17 00:00:00 2001 From: Akash Verma <138790903+akashverma0786@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:43:22 +0530 Subject: [PATCH] Fixes #19891 : Added measures in powerbi (#19990) --- .../source/dashboard/powerbi/metadata.py | 33 +++++++ .../source/dashboard/powerbi/models.py | 24 +++++ .../tests/unit/test_powerbi_table_measures.py | 91 +++++++++++++++++++ .../json/schema/entity/data/table.json | 4 +- .../src/generated/api/data/createContainer.ts | 2 + .../api/data/createDashboardDataModel.ts | 2 + .../ui/src/generated/api/data/createTable.ts | 2 + .../api/tests/createTestDefinition.ts | 2 + .../configuration/profilerConfiguration.ts | 2 + .../ui/src/generated/entity/data/container.ts | 2 + .../entity/data/dashboardDataModel.ts | 2 + .../ui/src/generated/entity/data/table.ts | 2 + .../storage/containerMetadataConfig.ts | 2 + .../storage/manifestMetadataConfig.ts | 2 + .../ui/src/generated/settings/settings.ts | 2 + .../ui/src/generated/tests/testDefinition.ts | 2 + 16 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 ingestion/tests/unit/test_powerbi_table_measures.py diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py index 61a001a5a1f..6806899dc6f 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py @@ -57,6 +57,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import ( Dataset, Group, PowerBIDashboard, + PowerBiMeasureModel, PowerBIReport, PowerBiTable, ) @@ -384,6 +385,35 @@ class PowerbiSource(DashboardServiceSource): ) ) + def _get_child_measures(self, table: PowerBiTable) -> List[Column]: + """ + Extract the measures of the table + """ + measures = [] + for measure in table.measures or []: + try: + measure_type = ( + DataType.MEASURE_HIDDEN + if measure.isHidden + else DataType.MEASURE_VISIBLE + ) + description_text = ( + f"{measure.description}\n\nExpression : {measure.expression}" + if measure.description + else f"Expression : {measure.expression}" + ) + parsed_measure = PowerBiMeasureModel( + dataType=measure_type, + dataTypeDisplay=measure_type, + name=measure.name, + description=description_text, + ) + measures.append(Column(**parsed_measure.model_dump())) + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning(f"Error processing datamodel nested measure: {err}") + return measures + def _get_child_columns(self, table: PowerBiTable) -> List[Column]: """ Extract the child columns from the fields @@ -423,8 +453,11 @@ class PowerbiSource(DashboardServiceSource): "description": table.description, } child_columns = self._get_child_columns(table=table) + child_measures = self._get_child_measures(table=table) if child_columns: parsed_table["children"] = child_columns + if child_measures: + parsed_table["children"].extend(child_measures) datasource_columns.append(Column(**parsed_table)) except Exception as exc: logger.debug(traceback.format_exc()) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py index 918c9e9a531..092cf5ccd45 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py @@ -98,6 +98,29 @@ class PowerBiColumns(BaseModel): description: Optional[str] = None +class PowerBiMeasureModel(BaseModel): + """ + Represents a Power BI measure, used before converting to a Column instance. + """ + + dataType: str + dataTypeDisplay: str + name: str + description: str + + +class PowerBiMeasures(BaseModel): + """ + PowerBI Column Model + Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-get-tables-in-group#measure + """ + + name: str + expression: str + description: Optional[str] = None + isHidden: bool + + class PowerBITableSource(BaseModel): """ PowerBI Table Source @@ -114,6 +137,7 @@ class PowerBiTable(BaseModel): name: str columns: Optional[List[PowerBiColumns]] = None + measures: Optional[List[PowerBiMeasures]] = None description: Optional[str] = None source: Optional[List[PowerBITableSource]] = None diff --git a/ingestion/tests/unit/test_powerbi_table_measures.py b/ingestion/tests/unit/test_powerbi_table_measures.py new file mode 100644 index 00000000000..84a2bea7b68 --- /dev/null +++ b/ingestion/tests/unit/test_powerbi_table_measures.py @@ -0,0 +1,91 @@ +import pytest + +from metadata.generated.schema.entity.data.table import Column, DataType +from metadata.generated.schema.type.basic import Markdown +from metadata.ingestion.source.dashboard.powerbi.metadata import PowerbiSource +from metadata.ingestion.source.dashboard.powerbi.models import ( + PowerBiMeasures, + PowerBiTable, +) + +test_cases = { + "visible_measure": { + "input": [ + PowerBiMeasures( + name="test_measure", + expression="SUM(Sales)", + description="Test Description", + isHidden=False, + ) + ], + "expected": [ + Column( + name="test_measure", + dataType=DataType.MEASURE_VISIBLE, + dataTypeDisplay=DataType.MEASURE_VISIBLE, + description=Markdown("Test Description\n\nExpression : SUM(Sales)"), + ) + ], + }, + "hidden_measure": { + "input": [ + PowerBiMeasures( + name="hidden_measure", + expression="AVG(Profit)", + description="Hidden", + isHidden=True, + ) + ], + "expected": [ + Column( + name="hidden_measure", + dataType=DataType.MEASURE_HIDDEN, + dataTypeDisplay=DataType.MEASURE_HIDDEN, + description=Markdown("Hidden\n\nExpression : AVG(Profit)"), + ) + ], + }, + "complex_expression": { + "input": [ + PowerBiMeasures( + name="complex_measure", + expression="SUM(Table[Column]) - SUM(OtherTable[OtherColumn])", + isHidden=False, + ) + ], + "expected": [ + Column( + name="complex_measure", + dataType=DataType.MEASURE_VISIBLE, + dataTypeDisplay=DataType.MEASURE_VISIBLE, + description=Markdown( + "Expression : SUM(Table[Column]) - SUM(OtherTable[OtherColumn])" + ), + ) + ], + }, +} + + +class MockPowerbiSource(PowerbiSource): + def __init__(self): + pass + + +@pytest.mark.parametrize("test_case_name, test_case", test_cases.items()) +def test_get_child_measures(test_case_name, test_case): + powerbi_source = MockPowerbiSource() + test_table = PowerBiTable( + name="test_table", + measures=test_case["input"], + ) + + result_columns = powerbi_source._get_child_measures(test_table) + + assert result_columns + + for expected_col, actual_col in zip(test_case["expected"], result_columns): + assert actual_col.name == expected_col.name + assert actual_col.dataType == expected_col.dataType + assert actual_col.dataTypeDisplay == expected_col.dataTypeDisplay + assert actual_col.description == expected_col.description diff --git a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json index c79b44d4e99..31ba356e277 100644 --- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json +++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json @@ -164,7 +164,9 @@ "BITMAP", "UINT", "BIT", - "MONEY" + "MONEY", + "MEASURE HIDDEN", + "MEASURE VISIBLE" ] }, "constraint": { diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts index f7cb8fff159..1d51262abdf 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts @@ -236,6 +236,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts index c50a325ca89..ad8ada7bb74 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts @@ -204,6 +204,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts index 419575fa60a..1dffafbc500 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts @@ -217,6 +217,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts index 2f17dc64149..a12d4f98093 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts @@ -242,6 +242,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts index fe3a515dfbf..be0fd64220e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts @@ -75,6 +75,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts index e1eb5092dbe..0fb42da5faa 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts @@ -532,6 +532,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts index aac3c28ce10..d64cfb0b936 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts @@ -419,6 +419,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts index 7d2dadec28b..81570e79f3b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts @@ -486,6 +486,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts index 90faddfd438..ceb22dedd42 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts @@ -182,6 +182,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts index 0eae7976079..ef78e35a41d 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts @@ -175,6 +175,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts index 26192f1bfa8..747918723b5 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts @@ -876,6 +876,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY", diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts index 82f716cb03b..8d42e3967e9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts @@ -371,6 +371,8 @@ export enum DataType { Lowcardinality = "LOWCARDINALITY", Macaddr = "MACADDR", Map = "MAP", + MeasureHidden = "MEASURE HIDDEN", + MeasureVisible = "MEASURE VISIBLE", Mediumblob = "MEDIUMBLOB", Mediumtext = "MEDIUMTEXT", Money = "MONEY",