diff --git a/.github/workflows/py-cli-e2e-tests.yml b/.github/workflows/py-cli-e2e-tests.yml index 759ecf466d4..8a504946170 100644 --- a/.github/workflows/py-cli-e2e-tests.yml +++ b/.github/workflows/py-cli-e2e-tests.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'vertica', 'python'] + e2e-test: ['bigquery', 'dbt_redshift', 'mssql', 'mysql', 'redash', 'snowflake', 'tableau', 'powerbi', 'vertica', 'python'] environment: test steps: @@ -111,6 +111,9 @@ jobs: E2E_REDASH_HOST_PORT: ${{ secrets.E2E_REDASH_HOST_PORT }} E2E_REDASH_USERNAME: ${{ secrets.E2E_REDASH_USERNAME }} E2E_REDASH_API_KEY: ${{ secrets.E2E_REDASH_API_KEY }} + E2E_POWERBI_CLIENT_SECRET: ${{ secrets.E2E_POWERBI_CLIENT_SECRET }} + E2E_POWERBI_CLIENT_ID: ${{ secrets.E2E_POWERBI_CLIENT_ID }} + E2E_POWERBI_TENANT_ID: ${{ secrets.E2E_POWERBI_TENANT_ID }} run: | source env/bin/activate export SITE_CUSTOMIZE_PATH=$(python -c "import site; import os; from pathlib import Path; print(os.path.relpath(site.getsitepackages()[0], str(Path.cwd())))")/sitecustomize.py diff --git a/ingestion/tests/cli_e2e/dashboard/powerbi/powerbi.yaml b/ingestion/tests/cli_e2e/dashboard/powerbi/powerbi.yaml new file mode 100644 index 00000000000..f163dc89303 --- /dev/null +++ b/ingestion/tests/cli_e2e/dashboard/powerbi/powerbi.yaml @@ -0,0 +1,30 @@ +source: + type: powerbi + serviceName: local_powerbi + serviceConnection: + config: + clientSecret: $E2E_POWERBI_CLIENT_SECRET + clientId: $E2E_POWERBI_CLIENT_ID + tenantId: $E2E_POWERBI_TENANT_ID + scope: + - https://analysis.windows.net/powerbi/api/.default + pagination_entity_per_page: 100 + useAdminApis: true + type: PowerBI + sourceConfig: + config: + type: DashboardMetadata + dashboardFilterPattern: {} + chartFilterPattern: {} + dbServiceNames: + - local_redshift +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/tests/cli_e2e/dashboard/powerbi/redshift.yaml b/ingestion/tests/cli_e2e/dashboard/powerbi/redshift.yaml new file mode 100644 index 00000000000..c65207f96e8 --- /dev/null +++ b/ingestion/tests/cli_e2e/dashboard/powerbi/redshift.yaml @@ -0,0 +1,25 @@ +source: + type: redshift + serviceName: local_redshift + serviceConnection: + config: + hostPort: $E2E_REDSHIFT_HOST_PORT + username: $E2E_REDSHIFT_USERNAME + password: $E2E_REDSHIFT_PASSWORD + database: $E2E_REDSHIFT_DATABASE + type: Redshift + sourceConfig: + config: + schemaFilterPattern: + includes: + - dbt_jaffle +sink: + type: metadata-rest + config: {} +workflowConfig: + loggerLevel: DEBUG + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXBiEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fNr3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3ud-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" diff --git a/ingestion/tests/cli_e2e/test_cli_powerbi.py b/ingestion/tests/cli_e2e/test_cli_powerbi.py new file mode 100644 index 00000000000..21bf89d94a0 --- /dev/null +++ b/ingestion/tests/cli_e2e/test_cli_powerbi.py @@ -0,0 +1,77 @@ +# Copyright 2022 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test PowerBI connector with CLI +""" +from pathlib import Path +from typing import List + +from .base.test_cli import PATH_TO_RESOURCES +from .common.test_cli_dashboard import CliCommonDashboard + + +class PowerBICliTest(CliCommonDashboard.TestSuite): + + # in case we want to do something before running the tests + def prepare(self) -> None: + redshift_file_path = str( + Path( + PATH_TO_RESOURCES + + f"/dashboard/{self.get_connector_name()}/redshift.yaml" + ) + ) + self.run_command(test_file_path=redshift_file_path) + + @staticmethod + def get_connector_name() -> str: + return "powerbi" + + def get_includes_dashboards(self) -> List[str]: + return [".*Supplier.*", ".*Lineage.*"] + + def get_excludes_dashboards(self) -> List[str]: + return ["Customer Profitability Sample"] + + def get_includes_charts(self) -> List[str]: + return ["Total Defect Quantity", "lineagetest", "lineagetest2work"] + + def get_excludes_charts(self) -> List[str]: + return ["Total Rejected Defect Quantity"] + + # PowerBI do not ingest datamodels + def get_includes_datamodels(self) -> List[str]: + return [] + + # PowerBI do not ingest datamodels + def get_excludes_datamodels(self) -> List[str]: + return [] + + def expected_entities(self) -> int: + return 74 + + def expected_lineage(self) -> int: + return 6 + + def expected_tags(self) -> int: + return 0 + + def expected_not_included_entities(self) -> int: + return 74 + + def expected_not_included_sink_entities(self) -> int: + return 81 + + def expected_filtered_mix(self) -> int: + return 14 + + def expected_filtered_sink_mix(self) -> int: + return 12 diff --git a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/PowerBI.md b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/PowerBI.md index 8eccd8dbaef..3487616accd 100644 --- a/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/PowerBI.md +++ b/openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/PowerBI.md @@ -2,8 +2,9 @@ In this section, we provide guides and references to use the PowerBI connector. -# Requirements -PowerBi Pro license is required to access the APIs +## Requirements + +To access the PowerBI APIs and import dashboards, charts, and datasets from PowerBI into OpenMetadata, a PowerBI Pro license is necessary. ## PowerBI Account Setup and Permissions @@ -26,9 +27,10 @@ Optional Permissions: (Without granting these permissions, the dataset informati - Dataset.Read.All - Dataset.ReadWrite.All -**Note**: +$$note Make sure that in the API permissions section **Tenant** related permissions are not being given to the app -Please refer [here](https://stackoverflow.com/questions/71001110/power-bi-rest-api-requests-not-authorizing-as-expected) for detailed explanation +Please refer [here](https://stackoverflow.com/questions/71001110/power-bi-rest-api-requests-not-authorizing-as-expected) for detailed explanation +$$ ### Step 3: Create New PowerBI workspace The service principal only works with [new workspaces](https://docs.microsoft.com/en-us/power-bi/collaborate-share/service-create-the-new-workspaces).