Fix 10689: Added Powerbi Datamodels (#11923)

ingestion/src/metadata/ingestion/source/dashboard/dashboard_service.py

@@ -65,6 +65,12 @@ from metadata.utils.logger import ingestion_logger
 
 logger = ingestion_logger()
 
+LINEAGE_MAP = {
+    Dashboard: "dashboard",
+    Table: "table",
+    DashboardDataModel: "dashboardDataModel",
+}
+
 
 class DashboardUsage(BaseModel):
     """
@@ -391,22 +397,18 @@ class DashboardServiceSource(TopologyRunnerMixin, Source, ABC):
     @staticmethod
     def _get_add_lineage_request(
         to_entity: Union[Dashboard, DashboardDataModel],
-        from_entity: Union[Table, DashboardDataModel],
+        from_entity: Union[Table, DashboardDataModel, Dashboard],
     ) -> Optional[AddLineageRequest]:
         if from_entity and to_entity:
             return AddLineageRequest(
                 edge=EntitiesEdge(
                     fromEntity=EntityReference(
                         id=from_entity.id.__root__,
-                        type="table"
-                        if isinstance(from_entity, Table)
-                        else "dashboardDataModel",
+                        type=LINEAGE_MAP[type(from_entity)],
                     ),
                     toEntity=EntityReference(
                         id=to_entity.id.__root__,
-                        type="dashboard"
-                        if isinstance(to_entity, Dashboard)
-                        else "dashboardDataModel",
+                        type=LINEAGE_MAP[type(to_entity)],
                     ),
                 )
             )

ingestion/src/metadata/ingestion/source/dashboard/powerbi/client.py

@@ -28,8 +28,10 @@ from metadata.ingestion.source.dashboard.powerbi.models import (
     Group,
     GroupsResponse,
     PowerBIDashboard,
+    PowerBIReport,
     PowerBiTable,
     PowerBiToken,
+    ReportsResponse,
     TablesResponse,
     Tile,
     TilesResponse,
@@ -130,6 +132,21 @@ class PowerBiApiClient:
 
         return None
 
+    def fetch_all_org_reports(self, group_id: str) -> Optional[List[PowerBIReport]]:
+        """Method to fetch all powerbi reports within the group
+        Returns:
+            List[PowerBIReport]
+        """
+        try:
+            response_data = self.client.get(f"/myorg/groups/{group_id}/reports")
+            response = ReportsResponse(**response_data)
+            return response.value
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error fetching group reports: {exc}")
+
+        return None
+
     def fetch_all_org_datasets(self, group_id: str) -> Optional[List[Dataset]]:
         """Method to fetch all powerbi datasets within the group
         Returns:

ingestion/src/metadata/ingestion/source/dashboard/powerbi/metadata.py

@@ -11,28 +11,49 @@
 """PowerBI source module"""
 
 import traceback
-from typing import Any, Iterable, List, Optional
+import uuid
+from typing import Any, Iterable, List, Optional, Union
 
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
+from metadata.generated.schema.api.data.createDashboardDataModel import (
+    CreateDashboardDataModelRequest,
+)
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
 from metadata.generated.schema.entity.data.chart import Chart, ChartType
-from metadata.generated.schema.entity.data.dashboard import Dashboard
-from metadata.generated.schema.entity.data.table import Table
+from metadata.generated.schema.entity.data.dashboard import Dashboard, DashboardType
+from metadata.generated.schema.entity.data.dashboardDataModel import (
+    DashboardDataModel,
+    DataModelType,
+)
+from metadata.generated.schema.entity.data.table import Column, DataType, Table
 from metadata.generated.schema.entity.services.connections.dashboard.powerBIConnection import (
     PowerBIConnection,
 )
 from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import (
     OpenMetadataConnection,
 )
+from metadata.generated.schema.entity.services.dashboardService import (
+    DashboardServiceType,
+)
 from metadata.generated.schema.metadataIngestion.workflow import (
     Source as WorkflowSource,
 )
 from metadata.ingestion.api.source import InvalidSourceException
 from metadata.ingestion.source.dashboard.dashboard_service import DashboardServiceSource
-from metadata.ingestion.source.dashboard.powerbi.models import Dataset, PowerBIDashboard
+from metadata.ingestion.source.dashboard.powerbi.models import (
+    Dataset,
+    PowerBIDashboard,
+    PowerBIReport,
+    PowerBiTable,
+)
+from metadata.ingestion.source.database.column_type_parser import ColumnTypeParser
 from metadata.utils import fqn
-from metadata.utils.filters import filter_by_chart, filter_by_dashboard
+from metadata.utils.filters import (
+    filter_by_chart,
+    filter_by_dashboard,
+    filter_by_datamodel,
+)
 from metadata.utils.helpers import clean_uri
 from metadata.utils.logger import ingestion_logger
 
@@ -84,6 +105,11 @@ class PowerbiSource(DashboardServiceSource):
                     or []
                 )
 
+            # add the reports to the groups
+            group.reports.extend(
+                self.client.fetch_all_org_reports(group_id=group.id) or []
+            )
+
             # add the datasets to the groups
             group.datasets.extend(
                 self.client.fetch_all_org_datasets(group_id=group.id) or []
@@ -181,25 +207,33 @@ class PowerbiSource(DashboardServiceSource):
                     continue
                 yield dashboard_details
 
-    def get_dashboards_list(self) -> Optional[List[PowerBIDashboard]]:
+    def get_dashboards_list(
+        self,
+    ) -> Optional[List[Union[PowerBIDashboard, PowerBIReport]]]:
         """
         Get List of all dashboards
         """
-        return self.context.workspace.dashboards
+        return self.context.workspace.reports + self.context.workspace.dashboards
 
-    def get_dashboard_name(self, dashboard: PowerBIDashboard) -> str:
+    def get_dashboard_name(
+        self, dashboard: Union[PowerBIDashboard, PowerBIReport]
+    ) -> str:
         """
         Get Dashboard Name
         """
-        return dashboard.displayName
+        if isinstance(dashboard, PowerBIDashboard):
+            return dashboard.displayName
+        return dashboard.name
 
-    def get_dashboard_details(self, dashboard: PowerBIDashboard) -> PowerBIDashboard:
+    def get_dashboard_details(
+        self, dashboard: Union[PowerBIDashboard, PowerBIReport]
+    ) -> Union[PowerBIDashboard, PowerBIReport]:
         """
         Get Dashboard Details
         """
         return dashboard
 
-    def get_dashboard_url(self, workspace_id: str, dashboard_id: str) -> str:
+    def _get_dashboard_url(self, workspace_id: str, dashboard_id: str) -> str:
         """
         Method to build the dashboard url
         """
@@ -208,7 +242,16 @@ class PowerbiSource(DashboardServiceSource):
             f"{workspace_id}/dashboards/{dashboard_id}"
         )
 
-    def get_chart_url(
+    def _get_report_url(self, workspace_id: str, dashboard_id: str) -> str:
+        """
+        Method to build the dashboard url
+        """
+        return (
+            f"{clean_uri(self.service_connection.hostPort)}/groups/"
+            f"{workspace_id}/reports/{dashboard_id}"
+        )
+
+    def _get_chart_url(
         self, report_id: Optional[str], workspace_id: str, dashboard_id: str
     ) -> str:
         """
@@ -222,74 +265,289 @@ class PowerbiSource(DashboardServiceSource):
             f"{workspace_id}/{chart_url_postfix}"
         )
 
+    def list_datamodels(self) -> Iterable[Dataset]:
+        """
+        Get All the Powerbi Datasets
+        """
+        if self.source_config.includeDataModels:
+            try:
+                for workspace in self.workspace_data:
+                    for dataset in workspace.datasets or []:
+                        if filter_by_datamodel(
+                            self.source_config.dataModelFilterPattern, dataset.name
+                        ):
+                            self.status.filter(dataset.name, "Data model filtered out.")
+                            continue
+                        yield dataset
+            except Exception as err:
+                logger.debug(traceback.format_exc())
+                logger.error(f"Unexpected error fetching PowerBI datasets - {err}")
+
+    def yield_bulk_datamodel(
+        self, dataset: Dataset
+    ) -> Iterable[CreateDashboardDataModelRequest]:
+        """
+        Method to fetch DataModels in bulk
+        """
+        try:
+            data_model_request = CreateDashboardDataModelRequest(
+                name=dataset.id,
+                displayName=dataset.name,
+                description=dataset.description,
+                service=self.context.dashboard_service.fullyQualifiedName.__root__,
+                dataModelType=DataModelType.PowerBIDataModel.value,
+                serviceType=DashboardServiceType.PowerBI.value,
+                columns=self._get_column_info(dataset),
+            )
+            yield data_model_request
+            self.status.scanned(f"Data Model Scanned: {data_model_request.displayName}")
+        except Exception as exc:
+            error_msg = f"Error yielding Data Model [{dataset.name}]: {exc}"
+            self.status.failed(
+                name=dataset.name,
+                error=error_msg,
+                stack_trace=traceback.format_exc(),
+            )
+            logger.error(error_msg)
+            logger.debug(traceback.format_exc())
+
+    def _get_child_columns(self, table: PowerBiTable) -> List[Column]:
+        """
+        Extract the child columns from the fields
+        """
+        columns = []
+        for column in table.columns or []:
+            try:
+                parsed_column = {
+                    "dataTypeDisplay": column.dataType
+                    if column.dataType
+                    else DataType.UNKNOWN.value,
+                    "dataType": ColumnTypeParser.get_column_type(
+                        column.dataType if column.dataType else None
+                    ),
+                    "name": str(uuid.uuid4()),
+                    "displayName": column.name,
+                }
+                if column.dataType and column.dataType == DataType.ARRAY.value:
+                    parsed_column["arrayDataType"] = DataType.UNKNOWN
+                columns.append(Column(**parsed_column))
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error processing datamodel nested column: {exc}")
+        return columns
+
+    def _get_column_info(self, dataset: Dataset) -> Optional[List[Column]]:
+        """
+        Args:
+            data_source: DataSource
+        Returns:
+            Columns details for Data Model
+        """
+        datasource_columns = []
+        for table in dataset.tables or []:
+            try:
+                parsed_table = {
+                    "dataTypeDisplay": "PowerBI Table",
+                    "dataType": DataType.TABLE,
+                    "name": str(uuid.uuid4()),
+                    "displayName": table.name,
+                    "description": table.description,
+                }
+                child_columns = self._get_child_columns(table=table)
+                if child_columns:
+                    parsed_table["children"] = child_columns
+                datasource_columns.append(Column(**parsed_table))
+            except Exception as exc:
+                logger.debug(traceback.format_exc())
+                logger.warning(f"Error to yield datamodel column: {exc}")
+        return datasource_columns
+
     def yield_dashboard(
-        self, dashboard_details: PowerBIDashboard
+        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
     ) -> Iterable[CreateDashboardRequest]:
         """
         Method to Get Dashboard Entity, Dashboard Charts & Lineage
         """
-        dashboard_request = CreateDashboardRequest(
-            name=dashboard_details.id,
-            dashboardUrl=self.get_dashboard_url(
-                workspace_id=self.context.workspace.id,
-                dashboard_id=dashboard_details.id,
-            ),
-            displayName=dashboard_details.displayName,
-            description="",
-            charts=[
-                fqn.build(
-                    self.metadata,
-                    entity_type=Chart,
-                    service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
-                    chart_name=chart.name.__root__,
+        try:
+            if isinstance(dashboard_details, PowerBIDashboard):
+                dashboard_request = CreateDashboardRequest(
+                    name=dashboard_details.id,
+                    dashboardUrl=self._get_dashboard_url(
+                        workspace_id=self.context.workspace.id,
+                        dashboard_id=dashboard_details.id,
+                    ),
+                    displayName=dashboard_details.displayName,
+                    dashboardType=DashboardType.Dashboard,
+                    charts=[
+                        fqn.build(
+                            self.metadata,
+                            entity_type=Chart,
+                            service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
+                            chart_name=chart.name.__root__,
+                        )
+                        for chart in self.context.charts
+                    ],
+                    service=self.context.dashboard_service.fullyQualifiedName.__root__,
                 )
-                for chart in self.context.charts
-            ],
-            service=self.context.dashboard_service.fullyQualifiedName.__root__,
-        )
-        yield dashboard_request
-        self.register_record(dashboard_request=dashboard_request)
+            else:
+                dashboard_request = CreateDashboardRequest(
+                    name=dashboard_details.id,
+                    dashboardType=DashboardType.Report,
+                    dashboardUrl=self._get_report_url(
+                        workspace_id=self.context.workspace.id,
+                        dashboard_id=dashboard_details.id,
+                    ),
+                    displayName=dashboard_details.name,
+                    service=self.context.dashboard_service.fullyQualifiedName.__root__,
+                )
+            yield dashboard_request
+            self.register_record(dashboard_request=dashboard_request)
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error creating dashboard [{dashboard_details}]: {exc}")
 
-    def yield_dashboard_lineage_details(
-        self, dashboard_details: PowerBIDashboard, db_service_name: str
-    ) -> Optional[Iterable[AddLineageRequest]]:
+    def create_report_dashboard_lineage(
+        self, dashboard_details: PowerBIDashboard
+    ) -> Iterable[CreateDashboardRequest]:
         """
-        Get lineage between dashboard and data sources
+        create lineage between report and dashboard
         """
         try:
             charts = dashboard_details.tiles
+            dashboard_fqn = fqn.build(
+                self.metadata,
+                entity_type=Dashboard,
+                service_name=self.config.serviceName,
+                dashboard_name=dashboard_details.id,
+            )
+            dashboard_entity = self.metadata.get_by_name(
+                entity=Dashboard,
+                fqn=dashboard_fqn,
+            )
             for chart in charts or []:
-                dataset = self.fetch_dataset_from_workspace(chart.datasetId)
-                if dataset:
-                    for table in dataset.tables:
-                        table_name = table.name
+                report = self._fetch_report_from_workspace(chart.reportId)
+                if report:
+                    report_fqn = fqn.build(
+                        self.metadata,
+                        entity_type=Dashboard,
+                        service_name=self.config.serviceName,
+                        dashboard_name=report.id,
+                    )
+                    report_entity = self.metadata.get_by_name(
+                        entity=Dashboard,
+                        fqn=report_fqn,
+                    )
 
-                        from_fqn = fqn.build(
-                            self.metadata,
-                            entity_type=Table,
-                            service_name=db_service_name,
-                            database_name=None,
-                            schema_name=None,
-                            table_name=table_name,
+                    if report_entity and dashboard_entity:
+                        yield self._get_add_lineage_request(
+                            to_entity=dashboard_entity, from_entity=report_entity
                         )
-                        from_entity = self.metadata.get_by_name(
-                            entity=Table,
-                            fqn=from_fqn,
-                        )
-                        to_fqn = fqn.build(
-                            self.metadata,
-                            entity_type=Dashboard,
-                            service_name=self.config.serviceName,
-                            dashboard_name=dashboard_details.id,
-                        )
-                        to_entity = self.metadata.get_by_name(
-                            entity=Dashboard,
-                            fqn=to_fqn,
-                        )
-                        if from_entity and to_entity:
-                            yield self._get_add_lineage_request(
-                                to_entity=to_entity, from_entity=from_entity
-                            )
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.error(f"Error to yield report and dashboard lineage details: {exc}")
+
+    def create_datamodel_report_lineage(
+        self, db_service_name: str, dashboard_details: PowerBIReport
+    ) -> Iterable[CreateDashboardRequest]:
+        """
+        create the lineage between datamodel and report
+        """
+        try:
+            report_fqn = fqn.build(
+                self.metadata,
+                entity_type=Dashboard,
+                service_name=self.config.serviceName,
+                dashboard_name=dashboard_details.id,
+            )
+            report_entity = self.metadata.get_by_name(
+                entity=Dashboard,
+                fqn=report_fqn,
+            )
+
+            dataset = self._fetch_dataset_from_workspace(dashboard_details.datasetId)
+            if dataset:
+                datamodel_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=DashboardDataModel,
+                    service_name=self.config.serviceName,
+                    data_model_name=dataset.id,
+                )
+                datamodel_entity = self.metadata.get_by_name(
+                    entity=DashboardDataModel,
+                    fqn=datamodel_fqn,
+                )
+
+                if datamodel_entity and report_entity:
+                    yield self._get_add_lineage_request(
+                        to_entity=report_entity, from_entity=datamodel_entity
+                    )
+
+                    # create the lineage between table and datamodel
+                    yield from self.create_table_datamodel_lineage(
+                        db_service_name=db_service_name,
+                        tables=dataset.tables,
+                        datamodel_entity=datamodel_entity,
+                    )
+        except Exception as exc:  # pylint: disable=broad-except
+            logger.debug(traceback.format_exc())
+            logger.error(
+                f"Error to yield datamodel and report lineage details for DB service name [{db_service_name}]: {exc}"
+            )
+
+    def create_table_datamodel_lineage(
+        self,
+        db_service_name: str,
+        tables: Optional[List[PowerBiTable]],
+        datamodel_entity: Optional[DashboardDataModel],
+    ):
+        """
+        Method to create lineage between table and datamodels
+        """
+        for table in tables or []:
+            try:
+                table_fqn = fqn.build(
+                    self.metadata,
+                    entity_type=Table,
+                    service_name=db_service_name,
+                    database_name=None,
+                    schema_name=None,
+                    table_name=table.name,
+                )
+                table_entity = self.metadata.get_by_name(
+                    entity=Table,
+                    fqn=table_fqn,
+                )
+
+                if table_entity and datamodel_entity:
+                    yield self._get_add_lineage_request(
+                        to_entity=datamodel_entity, from_entity=table_entity
+                    )
+            except Exception as exc:  # pylint: disable=broad-except
+                logger.debug(traceback.format_exc())
+                logger.error(
+                    f"Error to yield datamodel lineage details for DB service name [{db_service_name}]: {exc}"
+                )
+
+    def yield_dashboard_lineage_details(
+        self,
+        dashboard_details: Union[PowerBIDashboard, PowerBIReport],
+        db_service_name: str,
+    ) -> Iterable[AddLineageRequest]:
+        """
+        We will build the logic to build the logic as below
+        tables - datamodel - report - dashboard
+        """
+        try:
+            if isinstance(dashboard_details, PowerBIReport):
+                yield from self.create_datamodel_report_lineage(
+                    db_service_name=db_service_name, dashboard_details=dashboard_details
+                )
+
+            if isinstance(dashboard_details, PowerBIDashboard):
+                yield from self.create_report_dashboard_lineage(
+                    dashboard_details=dashboard_details
+                )
+
         except Exception as exc:  # pylint: disable=broad-except
             logger.debug(traceback.format_exc())
             logger.error(
@@ -297,7 +555,7 @@ class PowerbiSource(DashboardServiceSource):
             )
 
     def yield_dashboard_chart(
-        self, dashboard_details: PowerBIDashboard
+        self, dashboard_details: Union[PowerBIDashboard, PowerBIReport]
     ) -> Optional[Iterable[CreateChartRequest]]:
         """Get chart method
         Args:
@@ -305,37 +563,39 @@ class PowerbiSource(DashboardServiceSource):
         Returns:
             Iterable[Chart]
         """
-        charts = dashboard_details.tiles
-        for chart in charts or []:
-            try:
-                chart_title = chart.title
-                chart_display_name = chart_title if chart_title else chart.id
-                if filter_by_chart(
-                    self.source_config.chartFilterPattern, chart_display_name
-                ):
-                    self.status.filter(chart_display_name, "Chart Pattern not Allowed")
-                    continue
-                yield CreateChartRequest(
-                    name=chart.id,
-                    displayName=chart_display_name,
-                    description="",
-                    chartType=ChartType.Other.value,
-                    chartUrl=self.get_chart_url(
-                        report_id=chart.reportId,
-                        workspace_id=self.context.workspace.id,
-                        dashboard_id=dashboard_details.id,
-                    ),
-                    service=self.context.dashboard_service.fullyQualifiedName.__root__,
-                )
-                self.status.scanned(chart_display_name)
-            except Exception as exc:
-                name = chart.title
-                error = f"Error creating chart [{name}]: {exc}"
-                logger.debug(traceback.format_exc())
-                logger.warning(error)
-                self.status.failed(name, error, traceback.format_exc())
+        if isinstance(dashboard_details, PowerBIDashboard):
+            charts = dashboard_details.tiles
+            for chart in charts or []:
+                try:
+                    chart_title = chart.title
+                    chart_display_name = chart_title if chart_title else chart.id
+                    if filter_by_chart(
+                        self.source_config.chartFilterPattern, chart_display_name
+                    ):
+                        self.status.filter(
+                            chart_display_name, "Chart Pattern not Allowed"
+                        )
+                        continue
+                    yield CreateChartRequest(
+                        name=chart.id,
+                        displayName=chart_display_name,
+                        chartType=ChartType.Other.value,
+                        chartUrl=self._get_chart_url(
+                            report_id=chart.reportId,
+                            workspace_id=self.context.workspace.id,
+                            dashboard_id=dashboard_details.id,
+                        ),
+                        service=self.context.dashboard_service.fullyQualifiedName.__root__,
+                    )
+                    self.status.scanned(chart_display_name)
+                except Exception as exc:
+                    name = chart.title
+                    error = f"Error creating chart [{name}]: {exc}"
+                    logger.debug(traceback.format_exc())
+                    logger.warning(error)
+                    self.status.failed(name, error, traceback.format_exc())
 
-    def fetch_dataset_from_workspace(
+    def _fetch_dataset_from_workspace(
         self, dataset_id: Optional[str]
     ) -> Optional[Dataset]:
         """
@@ -352,3 +612,21 @@ class PowerbiSource(DashboardServiceSource):
             )
             return dataset_data
         return None
+
+    def _fetch_report_from_workspace(
+        self, report_id: Optional[str]
+    ) -> Optional[Dataset]:
+        """
+        Method to search the report using id in the workspace dict
+        """
+        if report_id:
+            report_data = next(
+                (
+                    report
+                    for report in self.context.workspace.reports or []
+                    if report.id == report_id
+                ),
+                None,
+            )
+            return report_data
+        return None

ingestion/src/metadata/ingestion/source/dashboard/powerbi/models.py

@@ -44,6 +44,17 @@ class PowerBIDashboard(BaseModel):
     tiles: Optional[List[Tile]] = []
 
 
+class PowerBIReport(BaseModel):
+    """
+    PowerBI PowerBIReport Model
+    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-report#report
+    """
+
+    id: str
+    name: str
+    datasetId: Optional[str]
+
+
 class DashboardsResponse(BaseModel):
     """
     PowerBI DashboardsResponse Model
@@ -54,6 +65,16 @@ class DashboardsResponse(BaseModel):
     value: List[PowerBIDashboard]
 
 
+class ReportsResponse(BaseModel):
+    """
+    PowerBI ReportsResponse Model
+    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/reports/get-reports-in-group
+    """
+
+    odata_context: str = Field(alias="@odata.context")
+    value: List[PowerBIReport]
+
+
 class TilesResponse(BaseModel):
     """
     PowerBI TilesResponse Model
@@ -64,6 +85,17 @@ class TilesResponse(BaseModel):
     value: List[Tile]
 
 
+class PowerBiColumns(BaseModel):
+    """
+    PowerBI Column Model
+    Definition: https://learn.microsoft.com/en-us/rest/api/power-bi/push-datasets/datasets-get-tables-in-group#column
+    """
+
+    name: str
+    dataType: Optional[str]
+    columnType: Optional[str]
+
+
 class PowerBiTable(BaseModel):
     """
     PowerBI Table Model
@@ -71,6 +103,8 @@ class PowerBiTable(BaseModel):
     """
 
     name: str
+    columns: Optional[List[PowerBiColumns]]
+    description: Optional[str]
 
 
 class TablesResponse(BaseModel):
@@ -92,6 +126,7 @@ class Dataset(BaseModel):
     id: str
     name: str
     tables: Optional[List[PowerBiTable]] = []
+    description: Optional[str]
 
 
 class DatasetResponse(BaseModel):
@@ -115,6 +150,7 @@ class Group(BaseModel):
     type: Optional[str]
     state: Optional[str]
     dashboards: Optional[List[PowerBIDashboard]] = []
+    reports: Optional[List[PowerBIReport]] = []
     datasets: Optional[List[Dataset]] = []
 
 

ingestion/tests/cli_e2e/test_cli_powerbi.py

@@ -56,22 +56,22 @@ class PowerBICliTest(CliCommonDashboard.TestSuite):
         return []
 
     def expected_entities(self) -> int:
-        return 74
+        return 91
 
     def expected_lineage(self) -> int:
-        return 6
+        return 58
 
     def expected_tags(self) -> int:
         return 0
 
     def expected_not_included_entities(self) -> int:
-        return 74
+        return 82
 
     def expected_not_included_sink_entities(self) -> int:
-        return 81
+        return 127
 
     def expected_filtered_mix(self) -> int:
-        return 14
+        return 19
 
     def expected_filtered_sink_mix(self) -> int:
-        return 12
+        return 36

openmetadata-docs/content/v1.1.0-snapshot/connectors/dashboard/powerbi/airflow.md

@@ -11,7 +11,7 @@ slug: /connectors/dashboard/powerbi/airflow
 | Charts     | {% icon iconName="check" /%} |
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
-| Datamodels | {% icon iconName="cross" /%} |
+| Datamodels | {% icon iconName="check" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the PowerBI connector.

openmetadata-docs/content/v1.1.0-snapshot/connectors/dashboard/powerbi/cli.md

@@ -11,7 +11,7 @@ slug: /connectors/dashboard/powerbi/cli
 | Charts     | {% icon iconName="check" /%} |
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
-| Datamodels | {% icon iconName="cross" /%} |
+| Datamodels | {% icon iconName="check" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the PowerBI connector.

openmetadata-docs/content/v1.1.0-snapshot/connectors/dashboard/powerbi/index.md

@@ -11,7 +11,7 @@ slug: /connectors/dashboard/powerbi
 | Charts     | {% icon iconName="check" /%} |
 | Owners     | {% icon iconName="cross" /%} |
 | Tags       | {% icon iconName="cross" /%} |
-| Datamodels | {% icon iconName="cross" /%} |
+| Datamodels | {% icon iconName="check" /%} |
 | Lineage    | {% icon iconName="check" /%} |
 
 In this section, we provide guides and references to use the PowerBI connector.
@@ -359,6 +359,22 @@ caption="View the Ingestion Pipeline from the Service Page" /%}
 
 {% /stepsContainer %}
 
+## PowerBI Lineage
+
+We are creating the lineage between `Tables - DataSources - Reports - Dashboards`.
+
+Below images show the lineage as shown in PowerBI and the lineage displayed in OpenMetadata
+
+{% image
+  src="/images/v1.1.0/connectors/powerbi/lineage-powerbi.png"
+  alt="PowerBI Lineage"
+  caption="Lineage in PowerBI" /%}
+
+{% image
+  src="/images/v1.1.0/connectors/powerbi/lineage-powerbi-om.png"
+  alt="PowerBI Lineage OpenMetadata"
+  caption="PowerBI Lineage in OpenMetadata" /%}
+
 ## Troubleshooting
 
  ### Workflow Deployment Error

openmetadata-docs/content/v1.1.0-snapshot/connectors/dashboard/powerbi/setup-powerbi-account.md

@@ -24,7 +24,7 @@ The required permissions are:
 - `Dashboard.Read.All`
 - `Dashboard.ReadWrite.All`
 
-Optional Permissions: (Without granting these permissions, the dataset information cannot be retrieved and the lineage processing will be skipped)
+Optional Permissions: (Without granting these permissions, the dataset information cannot be retrieved and the datamodel and lineage processing will be skipped)
 - `Dataset.Read.All`
 - `Dataset.ReadWrite.All`
 

openmetadata-service/src/main/java/org/openmetadata/service/resources/dashboards/DashboardResource.java

@@ -434,6 +434,7 @@ public class DashboardResource extends EntityResource<Dashboard, DashboardReposi
         .withCharts(getEntityReferences(Entity.CHART, create.getCharts()))
         .withDataModels(getEntityReferences(Entity.DASHBOARD_DATA_MODEL, create.getDataModels()))
         .withDashboardUrl(create.getDashboardUrl())
+        .withDashboardType(create.getDashboardType())
         .withTags(create.getTags());
   }
 }

openmetadata-spec/src/main/resources/json/schema/api/data/createDashboard.json

@@ -19,6 +19,9 @@
       "description": "Description of the database instance. What it has and how to use it.",
       "$ref": "../../type/basic.json#/definitions/markdown"
     },
+    "dashboardType": {
+      "$ref": "../../entity/data/dashboard.json#/definitions/dashboardType"
+    },
     "dashboardUrl": {
       "description": "Dashboard URL suffix from its service.",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/entity/data/dashboard.json

@@ -7,6 +7,26 @@
   "type": "object",
   "javaType": "org.openmetadata.schema.entity.data.Dashboard",
   "javaInterfaces": ["org.openmetadata.schema.EntityInterface"],
+  "definitions": {
+    "dashboardType": {
+      "javaType": "org.openmetadata.schema.type.DashboardType",
+      "description": "This schema defines the type used for describing different types of dashboards.",
+      "type": "string",
+      "default": "Dashboard",
+      "enum": [
+        "Dashboard",
+        "Report"
+      ],
+      "javaEnums": [
+        {
+          "name": "Dashboard"
+        },
+        {
+          "name": "Report"
+        }
+      ]
+    }
+  },
   "properties": {
     "id": {
       "description": "Unique identifier that identifies a dashboard instance.",
@@ -40,6 +60,9 @@
       "description": "User who made the update.",
       "type": "string"
     },
+    "dashboardType": {
+      "$ref": "#/definitions/dashboardType"
+    },
     "dashboardUrl": {
       "description": "Dashboard URL suffix from its service.",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/entity/data/dashboardDataModel.json

@@ -18,7 +18,8 @@
         "SupersetDataModel",
         "MetabaseDataModel",
         "LookMlView",
-        "LookMlExplore"
+        "LookMlExplore",
+        "PowerBIDataModel"
       ],
       "javaEnums": [
         {
@@ -35,6 +36,9 @@
         },
         {
           "name": "LookMlExplore"
+        },
+        {
+          "name": "PowerBIDataModel"
         }
       ]
     }