Added Powerbi E2E and docs (#11130)

.github/workflows/py-cli-e2e-tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'vertica', 'python']
+        e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python']
     environment: test
 
     steps:
@@ -111,6 +111,9 @@ jobs:
         E2E_REDASH_HOST_PORT: ${{ secrets.E2E_REDASH_HOST_PORT }}
         E2E_REDASH_USERNAME: ${{ secrets.E2E_REDASH_USERNAME }}
         E2E_REDASH_API_KEY: ${{ secrets.E2E_REDASH_API_KEY }}
+        E2E_POWERBI_CLIENT_SECRET: ${{ secrets.E2E_POWERBI_CLIENT_SECRET }}
+        E2E_POWERBI_CLIENT_ID: ${{ secrets.E2E_POWERBI_CLIENT_ID }}
+        E2E_POWERBI_TENANT_ID: ${{ secrets.E2E_POWERBI_TENANT_ID }}
       run: |
         source env/bin/activate
         export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py

ingestion/tests/cli_e2e/dashboard/powerbi/powerbi.yaml

@@ -0,0 +1,30 @@
+source:
+  type: powerbi
+  serviceName: local_powerbi
+  serviceConnection:
+    config:
+      clientSecret: $E2E_POWERBI_CLIENT_SECRET
+      clientId: $E2E_POWERBI_CLIENT_ID
+      tenantId: $E2E_POWERBI_TENANT_ID
+      scope:
+        - https://analysis.windows.net/powerbi/api/.default
+      pagination_entity_per_page: 100
+      useAdminApis: true
+      type: PowerBI
+  sourceConfig:
+    config:
+      type: DashboardMetadata
+      dashboardFilterPattern: {}
+      chartFilterPattern: {}
+      dbServiceNames:
+      - local_redshift
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/tests/cli_e2e/dashboard/powerbi/redshift.yaml

@@ -0,0 +1,25 @@
+source:
+  type: redshift
+  serviceName: local_redshift
+  serviceConnection:
+    config:
+      hostPort: $E2E_REDSHIFT_HOST_PORT
+      username: $E2E_REDSHIFT_USERNAME
+      password: $E2E_REDSHIFT_PASSWORD
+      database: $E2E_REDSHIFT_DATABASE
+      type: Redshift
+  sourceConfig:
+    config:
+      schemaFilterPattern:
+        includes:
+        - dbt_jaffle
+sink:
+  type: metadata-rest
+  config: {}
+workflowConfig:
+  loggerLevel: DEBUG
+  openMetadataServerConfig:
+    hostPort: http://localhost:8585/api
+    authProvider: openmetadata
+    securityConfig:
+      "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"

ingestion/tests/cli_e2e/test_cli_powerbi.py

@@ -0,0 +1,77 @@
+#  Copyright 2022 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+"""
+Test PowerBI connector with CLI
+"""
+from pathlib import Path
+from typing import List
+
+from .base.test_cli import PATH_TO_RESOURCES
+from .common.test_cli_dashboard import CliCommonDashboard
+
+
+class PowerBICliTest(CliCommonDashboard.TestSuite):
+
+    # in case we want to do something before running the tests
+    def prepare(self) -> None:
+        redshift_file_path = str(
+            Path(
+                PATH_TO_RESOURCES
+                + f"/dashboard/{self.get_connector_name()}/redshift.yaml"
+            )
+        )
+        self.run_command(test_file_path=redshift_file_path)
+
+    @staticmethod
+    def get_connector_name() -> str:
+        return "powerbi"
+
+    def get_includes_dashboards(self) -> List[str]:
+        return [".*Supplier.*", ".*Lineage.*"]
+
+    def get_excludes_dashboards(self) -> List[str]:
+        return ["Customer Profitability Sample"]
+
+    def get_includes_charts(self) -> List[str]:
+        return ["Total Defect Quantity", "lineagetest", "lineagetest2work"]
+
+    def get_excludes_charts(self) -> List[str]:
+        return ["Total Rejected Defect Quantity"]
+
+    # PowerBI do not ingest datamodels
+    def get_includes_datamodels(self) -> List[str]:
+        return []
+
+    # PowerBI do not ingest datamodels
+    def get_excludes_datamodels(self) -> List[str]:
+        return []
+
+    def expected_entities(self) -> int:
+        return 74
+
+    def expected_lineage(self) -> int:
+        return 6
+
+    def expected_tags(self) -> int:
+        return 0
+
+    def expected_not_included_entities(self) -> int:
+        return 74
+
+    def expected_not_included_sink_entities(self) -> int:
+        return 81
+
+    def expected_filtered_mix(self) -> int:
+        return 14
+
+    def expected_filtered_sink_mix(self) -> int:
+        return 12

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/PowerBI.md

@@ -2,8 +2,9 @@
 
 In this section, we provide guides and references to use the PowerBI connector.
 
-# Requirements
-PowerBi Pro license is required to access the APIs
+## Requirements
+
+To access the PowerBI APIs and import dashboards, charts, and datasets from PowerBI into OpenMetadata, a PowerBI Pro license is necessary.
 
 ## PowerBI Account Setup and Permissions
 
@@ -26,9 +27,10 @@ Optional Permissions: (Without granting these permissions, the dataset informati
 - Dataset.Read.All
 - Dataset.ReadWrite.All
 
-**Note**:
+$$note
 Make sure that in the API permissions section **Tenant** related permissions are not being given to the app
-Please refer [here](https://stackoverflow.com/questions/71001110/power-bi-rest-api-requests-not-authorizing-as-expected) for detailed explanation 
+Please refer [here](https://stackoverflow.com/questions/71001110/power-bi-rest-api-requests-not-authorizing-as-expected) for detailed explanation
+$$
 
 ### Step 3: Create New PowerBI workspace
 The service principal only works with [new workspaces](https://docs.microsoft.com/en-us/power-bi/collaborate-share/service-create-the-new-workspaces).