Fixes #19891 : Added measures in powerbi (#19990)

2025-12-04 19:44:58 +00:00 · 2025-03-17 14:43:22 +05:30 · 2025-03-17 14:43:22 +05:30 · cf7a442e32
commit cf7a442e32
parent c51e0ca606
16 changed files with 175 additions and 1 deletions
--- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py
@ -57,6 +57,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
    Dataset,
    Group,
    PowerBIDashboard,
+    PowerBiMeasureModel,
    PowerBIReport,
    PowerBiTable,
 )
@ -384,6 +385,35 @@ class PowerbiSource(DashboardServiceSource):
                )
            )

+    def _get_child_measures(self, table: PowerBiTable) -> List[Column]:
+        """
+        Extract the measures of the table
+        """
+        measures = []
+        for measure in table.measures or []:
+            try:
+                measure_type = (
+                    DataType.MEASURE_HIDDEN
+                    if measure.isHidden
+                    else DataType.MEASURE_VISIBLE
+                )
+                description_text = (
+                    f"{measure.description}\n\nExpression : {measure.expression}"
+                    if measure.description
+                    else f"Expression : {measure.expression}"
+                )
+                parsed_measure = PowerBiMeasureModel(
+                    dataType=measure_type,
+                    dataTypeDisplay=measure_type,
+                    name=measure.name,
+                    description=description_text,
+                )
+                measures.append(Column(**parsed_measure.model_dump()))
+            except Exception as err:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error processing datamodel nested measure: {err}")
+        return measures
+
    def _get_child_columns(self, table: PowerBiTable) -> List[Column]:
        """
        Extract the child columns from the fields
@ -423,8 +453,11 @@ class PowerbiSource(DashboardServiceSource):
                    "description": table.description,
                }
                child_columns = self._get_child_columns(table=table)
+                child_measures = self._get_child_measures(table=table)
                if child_columns:
                    parsed_table["children"] = child_columns
+                if child_measures:
+                    parsed_table["children"].extend(child_measures)
                datasource_columns.append(Column(**parsed_table))
            except Exception as exc:
                logger.debug(traceback.format_exc())
--- a/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py
@ -98,6 +98,29 @@ class PowerBiColumns(BaseModel):
    description: Optional[str] = None


+class PowerBiMeasureModel(BaseModel):
+    """
+    Represents a Power BI measure, used before converting to a Column instance.
+    """
+
+    dataType: str
+    dataTypeDisplay: str
+    name: str
+    description: str
+
+
+class PowerBiMeasures(BaseModel):
+    """
+    PowerBI Column Model
+    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-get-tables-in-group#measure
+    """
+
+    name: str
+    expression: str
+    description: Optional[str] = None
+    isHidden: bool
+
+
 class PowerBITableSource(BaseModel):
    """
    PowerBI Table Source
@ -114,6 +137,7 @@ class PowerBiTable(BaseModel):

    name: str
    columns: Optional[List[PowerBiColumns]] = None
+    measures: Optional[List[PowerBiMeasures]] = None
    description: Optional[str] = None
    source: Optional[List[PowerBITableSource]] = None

--- a/ingestion/tests/unit/test_powerbi_table_measures.py
+++ b/ingestion/tests/unit/test_powerbi_table_measures.py
@ -0,0 +1,91 @@
+import pytest
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+from metadata.generated.schema.type.basic import Markdown
+from metadata.ingestion.source.dashboard.powerbi.metadata import PowerbiSource
+from metadata.ingestion.source.dashboard.powerbi.models import (
+    PowerBiMeasures,
+    PowerBiTable,
+)
+
+test_cases = {
+    "visible_measure": {
+        "input": [
+            PowerBiMeasures(
+                name="test_measure",
+                expression="SUM(Sales)",
+                description="Test Description",
+                isHidden=False,
+            )
+        ],
+        "expected": [
+            Column(
+                name="test_measure",
+                dataType=DataType.MEASURE_VISIBLE,
+                dataTypeDisplay=DataType.MEASURE_VISIBLE,
+                description=Markdown("Test Description\n\nExpression : SUM(Sales)"),
+            )
+        ],
+    },
+    "hidden_measure": {
+        "input": [
+            PowerBiMeasures(
+                name="hidden_measure",
+                expression="AVG(Profit)",
+                description="Hidden",
+                isHidden=True,
+            )
+        ],
+        "expected": [
+            Column(
+                name="hidden_measure",
+                dataType=DataType.MEASURE_HIDDEN,
+                dataTypeDisplay=DataType.MEASURE_HIDDEN,
+                description=Markdown("Hidden\n\nExpression : AVG(Profit)"),
+            )
+        ],
+    },
+    "complex_expression": {
+        "input": [
+            PowerBiMeasures(
+                name="complex_measure",
+                expression="SUM(Table[Column]) - SUM(OtherTable[OtherColumn])",
+                isHidden=False,
+            )
+        ],
+        "expected": [
+            Column(
+                name="complex_measure",
+                dataType=DataType.MEASURE_VISIBLE,
+                dataTypeDisplay=DataType.MEASURE_VISIBLE,
+                description=Markdown(
+                    "Expression : SUM(Table[Column]) - SUM(OtherTable[OtherColumn])"
+                ),
+            )
+        ],
+    },
+}
+
+
+class MockPowerbiSource(PowerbiSource):
+    def __init__(self):
+        pass
+
+
+@pytest.mark.parametrize("test_case_name, test_case", test_cases.items())
+def test_get_child_measures(test_case_name, test_case):
+    powerbi_source = MockPowerbiSource()
+    test_table = PowerBiTable(
+        name="test_table",
+        measures=test_case["input"],
+    )
+
+    result_columns = powerbi_source._get_child_measures(test_table)
+
+    assert result_columns
+
+    for expected_col, actual_col in zip(test_case["expected"], result_columns):
+        assert actual_col.name == expected_col.name
+        assert actual_col.dataType == expected_col.dataType
+        assert actual_col.dataTypeDisplay == expected_col.dataTypeDisplay
+        assert actual_col.description == expected_col.description
--- a/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json
+++ b/openmetadata-spec/src/main/resources/json/schema/entity/data/table.json
@ -164,7 +164,9 @@
        "BITMAP",
        "UINT",
        "BIT",
-        "MONEY"
+        "MONEY",
+        "MEASURE HIDDEN",
+        "MEASURE VISIBLE"
      ]
    },
    "constraint": {
--- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts
@ -236,6 +236,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts
@ -204,6 +204,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts
@ -217,6 +217,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts
@ -242,6 +242,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts
@ -75,6 +75,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts
@ -532,6 +532,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts
@ -419,6 +419,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts
@ -486,6 +486,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts
@ -182,6 +182,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts
@ -175,6 +175,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts
@ -876,6 +876,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",
--- a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts
+++ b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts
@ -371,6 +371,8 @@ export enum DataType {
    Lowcardinality = "LOWCARDINALITY",
    Macaddr = "MACADDR",
    Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
    Mediumblob = "MEDIUMBLOB",
    Mediumtext = "MEDIUMTEXT",
    Money = "MONEY",