Fixes #19891 : Added measures in powerbi (#19990)

ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -57,6 +57,7 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
     Dataset,
     Group,
     PowerBIDashboard,
+    PowerBiMeasureModel,
     PowerBIReport,
     PowerBiTable,
 )
@@ -384,6 +385,35 @@ class PowerbiSource(DashboardServiceSource):
                 )
             )
 
+    def _get_child_measures(self, table: PowerBiTable) -> List[Column]:
+        """
+        Extract the measures of the table
+        """
+        measures = []
+        for measure in table.measures or []:
+            try:
+                measure_type = (
+                    DataType.MEASURE_HIDDEN
+                    if measure.isHidden
+                    else DataType.MEASURE_VISIBLE
+                )
+                description_text = (
+                    f"{measure.description}\n\nExpression : {measure.expression}"
+                    if measure.description
+                    else f"Expression : {measure.expression}"
+                )
+                parsed_measure = PowerBiMeasureModel(
+                    dataType=measure_type,
+                    dataTypeDisplay=measure_type,
+                    name=measure.name,
+                    description=description_text,
+                )
+                measures.append(Column(**parsed_measure.model_dump()))
+            except Exception as err:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error processing datamodel nested measure: {err}")
+        return measures
+
     def _get_child_columns(self, table: PowerBiTable) -> List[Column]:
         """
         Extract the child columns from the fields
@@ -423,8 +453,11 @@ class PowerbiSource(DashboardServiceSource):
                     "description": table.description,
                 }
                 child_columns = self._get_child_columns(table=table)
+                child_measures = self._get_child_measures(table=table)
                 if child_columns:
                     parsed_table["children"] = child_columns
+                if child_measures:
+                    parsed_table["children"].extend(child_measures)
                 datasource_columns.append(Column(**parsed_table))
             except Exception as exc:
                 logger.debug(traceback.format_exc())

ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py

@@ -98,6 +98,29 @@ class PowerBiColumns(BaseModel):
     description: Optional[str] = None
 
 
+class PowerBiMeasureModel(BaseModel):
+    """
+    Represents a Power BI measure, used before converting to a Column instance.
+    """
+
+    dataType: str
+    dataTypeDisplay: str
+    name: str
+    description: str
+
+
+class PowerBiMeasures(BaseModel):
+    """
+    PowerBI Column Model
+    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-get-tables-in-group#measure
+    """
+
+    name: str
+    expression: str
+    description: Optional[str] = None
+    isHidden: bool
+
+
 class PowerBITableSource(BaseModel):
     """
     PowerBI Table Source
@@ -114,6 +137,7 @@ class PowerBiTable(BaseModel):
 
     name: str
     columns: Optional[List[PowerBiColumns]] = None
+    measures: Optional[List[PowerBiMeasures]] = None
     description: Optional[str] = None
     source: Optional[List[PowerBITableSource]] = None
 

ingestion/tests/unit/test_powerbi_table_measures.py

@@ -0,0 +1,91 @@
+import pytest
+
+from metadata.generated.schema.entity.data.table import Column, DataType
+from metadata.generated.schema.type.basic import Markdown
+from metadata.ingestion.source.dashboard.powerbi.metadata import PowerbiSource
+from metadata.ingestion.source.dashboard.powerbi.models import (
+    PowerBiMeasures,
+    PowerBiTable,
+)
+
+test_cases = {
+    "visible_measure": {
+        "input": [
+            PowerBiMeasures(
+                name="test_measure",
+                expression="SUM(Sales)",
+                description="Test Description",
+                isHidden=False,
+            )
+        ],
+        "expected": [
+            Column(
+                name="test_measure",
+                dataType=DataType.MEASURE_VISIBLE,
+                dataTypeDisplay=DataType.MEASURE_VISIBLE,
+                description=Markdown("Test Description\n\nExpression : SUM(Sales)"),
+            )
+        ],
+    },
+    "hidden_measure": {
+        "input": [
+            PowerBiMeasures(
+                name="hidden_measure",
+                expression="AVG(Profit)",
+                description="Hidden",
+                isHidden=True,
+            )
+        ],
+        "expected": [
+            Column(
+                name="hidden_measure",
+                dataType=DataType.MEASURE_HIDDEN,
+                dataTypeDisplay=DataType.MEASURE_HIDDEN,
+                description=Markdown("Hidden\n\nExpression : AVG(Profit)"),
+            )
+        ],
+    },
+    "complex_expression": {
+        "input": [
+            PowerBiMeasures(
+                name="complex_measure",
+                expression="SUM(Table[Column]) - SUM(OtherTable[OtherColumn])",
+                isHidden=False,
+            )
+        ],
+        "expected": [
+            Column(
+                name="complex_measure",
+                dataType=DataType.MEASURE_VISIBLE,
+                dataTypeDisplay=DataType.MEASURE_VISIBLE,
+                description=Markdown(
+                    "Expression : SUM(Table[Column]) - SUM(OtherTable[OtherColumn])"
+                ),
+            )
+        ],
+    },
+}
+
+
+class MockPowerbiSource(PowerbiSource):
+    def __init__(self):
+        pass
+
+
+@pytest.mark.parametrize("test_case_name, test_case", test_cases.items())
+def test_get_child_measures(test_case_name, test_case):
+    powerbi_source = MockPowerbiSource()
+    test_table = PowerBiTable(
+        name="test_table",
+        measures=test_case["input"],
+    )
+
+    result_columns = powerbi_source._get_child_measures(test_table)
+
+    assert result_columns
+
+    for expected_col, actual_col in zip(test_case["expected"], result_columns):
+        assert actual_col.name == expected_col.name
+        assert actual_col.dataType == expected_col.dataType
+        assert actual_col.dataTypeDisplay == expected_col.dataTypeDisplay
+        assert actual_col.description == expected_col.description

openmetadata-spec/src/main/resources/json/schema/entity/data/table.json

@@ -164,7 +164,9 @@
         "BITMAP",
         "UINT",
         "BIT",
-        "MONEY"
+        "MONEY",
+        "MEASURE HIDDEN",
+        "MEASURE VISIBLE"
       ]
     },
     "constraint": {

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createContainer.ts

@@ -236,6 +236,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createDashboardDataModel.ts

@@ -204,6 +204,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/api/data/createTable.ts

@@ -217,6 +217,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestDefinition.ts

@@ -242,6 +242,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/configuration/profilerConfiguration.ts

@@ -75,6 +75,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/entity/data/container.ts

@@ -532,6 +532,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/entity/data/dashboardDataModel.ts

@@ -419,6 +419,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/entity/data/table.ts

@@ -486,6 +486,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/containerMetadataConfig.ts

@@ -182,6 +182,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/storage/manifestMetadataConfig.ts

@@ -175,6 +175,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/settings/settings.ts

@@ -876,6 +876,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",

openmetadata-ui/src/main/resources/ui/src/generated/tests/testDefinition.ts

@@ -371,6 +371,8 @@ export enum DataType {
     Lowcardinality = "LOWCARDINALITY",
     Macaddr = "MACADDR",
     Map = "MAP",
+    MeasureHidden = "MEASURE HIDDEN",
+    MeasureVisible = "MEASURE VISIBLE",
     Mediumblob = "MEDIUMBLOB",
     Mediumtext = "MEDIUMTEXT",
     Money = "MONEY",