yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-30 03:56:04 +00:00
Code Issues Packages Projects Releases Wiki Activity
datahub/metadata-ingestion/tests/integration/powerbi/test_powerbi.py

1826 lines
68 KiB
Python
Raw Blame History

import datetime
import logging
import re
import sys
from typing import Any, Dict, List, Optional, cast
from unittest import mock
from unittest.mock import MagicMock
import pytest
from freezegun import freeze_time
from datahub.ingestion.run.pipeline import Pipeline
from datahub.ingestion.source.powerbi.config import (
Constant,
PowerBiDashboardSourceConfig,
SupportedDataPlatform,
)
from datahub.ingestion.source.powerbi.powerbi import PowerBiDashboardSource
from datahub.ingestion.source.powerbi.rest_api_wrapper.data_classes import (
Page,
Report,
Workspace,
)
from tests.test_helpers import mce_helpers, test_connection_helpers
pytestmark = pytest.mark.integration_batch_2
FROZEN_TIME = "2022-02-03 07:00:00"
def enable_logging():
# set logging to console
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
logging.getLogger().setLevel(logging.DEBUG)
class MsalClient:
call_num = 0
token: Dict[str, Any] = {
"access_token": "dummy",
}
@staticmethod
def acquire_token_for_client(*args, **kwargs):
MsalClient.call_num += 1
return MsalClient.token
@staticmethod
def reset():
MsalClient.call_num = 0
def mock_msal_cca(*args, **kwargs):
return MsalClient()
def scan_init_response(request, context):
# Request mock is passing POST input in the form of workspaces=<workspace_id>
# If we scan 2 or more, it get messy like this. 'workspaces=64ED5CAD-7C10-4684-8180-826122881108&workspaces=64ED5CAD-7C22-4684-8180-826122881108'
workspace_id_list = request.text.replace("&", "").split("workspaces=")
workspace_id = "||".join(workspace_id_list[1:])
w_id_vs_response: Dict[str, Any] = {
"64ED5CAD-7C10-4684-8180-826122881108": {
"id": "4674efd1-603c-4129-8d82-03cf2be05aff"
},
"64ED5CAD-7C22-4684-8180-826122881108": {
"id": "a674efd1-603c-4129-8d82-03cf2be05aff"
},
"64ED5CAD-7C10-4684-8180-826122881108||64ED5CAD-7C22-4684-8180-826122881108": {
"id": "a674efd1-603c-4129-8d82-03cf2be05aff"
},
}
return w_id_vs_response[workspace_id]
def register_mock_api(request_mock: Any, override_data: Optional[dict] = None) -> None:
override_data = override_data or {}
api_vs_response = {
"https://api.powerbi.com/v1.0/myorg/groups": {
"method": "GET",
"status_code": 200,
"json": {
"@odata.count": 3,
"value": [
{
"id": "64ED5CAD-7C10-4684-8180-826122881108",
"isReadOnly": True,
"name": "demo-workspace",
"type": "Workspace",
},
{
"id": "64ED5CAD-7C22-4684-8180-826122881108",
"isReadOnly": True,
"name": "second-demo-workspace",
"type": "Workspace",
},
{
"id": "64ED5CAD-7322-4684-8180-826122881108",
"isReadOnly": True,
"name": "Workspace 2",
"type": "Workspace",
},
],
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
"isReadOnly": True,
"displayName": "test_dashboard",
"description": "Description of test dashboard",
"embedUrl": "https://localhost/dashboards/embed/1",
"webUrl": "https://localhost/dashboards/web/1",
}
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE",
"isReadOnly": True,
"displayName": "test_dashboard2",
"embedUrl": "https://localhost/dashboards/embed/1",
"webUrl": "https://localhost/dashboards/web/1",
}
]
},
},
"https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"identifier": "User1@foo.com",
"displayName": "user1",
"emailAddress": "User1@foo.com",
"datasetUserAccessRight": "ReadWrite",
"graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
"principalType": "User",
},
{
"identifier": "User2@foo.com",
"displayName": "user2",
"emailAddress": "User2@foo.com",
"datasetUserAccessRight": "ReadWrite",
"graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
"principalType": "User",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"identifier": "User1@foo.com",
"displayName": "user1",
"emailAddress": "User1@foo.com",
"datasetUserAccessRight": "ReadWrite",
"graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
"principalType": "User",
},
{
"identifier": "User2@foo.com",
"displayName": "user2",
"emailAddress": "User2@foo.com",
"datasetUserAccessRight": "ReadWrite",
"graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
"principalType": "User",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/users": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"identifier": "User3@foo.com",
"displayName": "user3",
"emailAddress": "User3@foo.com",
"datasetUserAccessRight": "ReadWrite",
"graphId": "C9EE53F2-88EA-4711-A173-AF0515A3CD46",
"principalType": "User",
},
{
"identifier": "User4@foo.com",
"displayName": "user4",
"emailAddress": "User4@foo.com",
"datasetUserAccessRight": "ReadWrite",
"graphId": "C9EE53F2-88EA-4711-A173-AF0515A5REWS",
"principalType": "User",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"id": "B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0",
"title": "test_tile",
"embedUrl": "https://localhost/tiles/embed/1",
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
},
{
"id": "23212598-23b5-4980-87cc-5fc0ecd84385",
"title": "yearly_sales",
"embedUrl": "https://localhost/tiles/embed/2",
"datasetId": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles": {
"method": "GET",
"status_code": 200,
"json": {"value": []},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445": {
"method": "GET",
"status_code": 200,
"json": {
"id": "05169CD2-E713-41E6-9600-1D8066D95445",
"name": "library-dataset",
"description": "Library dataset description",
"webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445",
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445": {
"method": "GET",
"status_code": 200,
"json": {
"id": "05169CD2-E713-41E6-96AA-1D8066D95445",
"name": "library-dataset",
"description": "Library dataset description",
"webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445",
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed": {
"method": "GET",
"status_code": 200,
"json": {
"id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
"name": "hr_pbi_test",
"description": "hr pbi test description",
"webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed",
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
"datasourceType": "PostgreSql",
"connectionDetails": {
"database": "library_db",
"server": "foo",
},
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445/datasources": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"datasourceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
"datasourceType": "PostgreSql",
"connectionDetails": {
"database": "library_db",
"server": "foo",
},
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff": {
"method": "GET",
"status_code": 200,
"json": {
"status": "SUCCEEDED",
},
},
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff": {
"method": "GET",
"status_code": 200,
"json": {
"status": "SUCCEEDED",
},
},
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": {
"method": "GET",
"status_code": 200,
"json": {
"workspaces": [
{
"id": "64ED5CAD-7C10-4684-8180-826122881108",
"name": "demo-workspace",
"state": "Active",
"datasets": [
{
"id": "05169CD2-E713-41E6-9600-1D8066D95445",
"endorsementDetails": {"endorsement": "Promoted"},
"name": "test_sf_pbi_test",
"tables": [
{
"name": "public issue_history",
"source": [
{
"expression": "dummy",
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "SNOWFLAKE_TESTTABLE",
"source": [
{
"expression": 'let\n Source = Snowflake.Databases("hp123rt5.ap-southeast-2.fakecomputing.com","PBI_TEST_WAREHOUSE_PROD",[Role="PBI_TEST_MEMBER"]),\n PBI_TEST_Database = Source{[Name="PBI_TEST",Kind="Database"]}[Data],\n TEST_Schema = PBI_TEST_Database{[Name="TEST",Kind="Schema"]}[Data],\n TESTTABLE_Table = TEST_Schema{[Name="TESTTABLE",Kind="Table"]}[Data]\nin\n TESTTABLE_Table',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "snowflake native-query",
"source": [
{
"expression": 'let\n Source = Value.NativeQuery(Snowflake.Databases("bu20658.ap-southeast-2.snowflakecomputing.com","operations_analytics_warehouse_prod",[Role="OPERATIONS_ANALYTICS_MEMBER"]){[Name="OPERATIONS_ANALYTICS"]}[Data], "SELECT#(lf)concat((UPPER(REPLACE(SELLER,\'-\',\'\'))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,\'-\',\'\'))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4", null, [EnableFolding=true]),\n #"Added Conditional Column" = Table.AddColumn(Source, "SME Units ENT", each if [DEAL_TYPE] = "SME Unit" then [UNIT] else 0),\n #"Added Conditional Column1" = Table.AddColumn(#"Added Conditional Column", "Banklink Units", each if [DEAL_TYPE] = "Banklink" then [UNIT] else 0),\n #"Removed Columns" = Table.RemoveColumns(#"Added Conditional Column1",{"Banklink Units"}),\n #"Added Custom" = Table.AddColumn(#"Removed Columns", "Banklink Units", each if [DEAL_TYPE] = "Banklink" and [SALES_TYPE] = "3 - Upsell"\nthen [UNIT]\n\nelse if [SALES_TYPE] = "Adjusted BL Migration"\nthen [UNIT]\n\nelse 0),\n #"Added Custom1" = Table.AddColumn(#"Added Custom", "SME Units in $ (*$361)", each if [DEAL_TYPE] = "SME Unit" \nand [SALES_TYPE] <> "4 - Renewal"\n then [UNIT] * 361\nelse 0),\n #"Added Custom2" = Table.AddColumn(#"Added Custom1", "Banklink in $ (*$148)", each [Banklink Units] * 148)\nin\n #"Added Custom2"',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "big-query-with-parameter",
"source": [
{
"expression": 'let\n Source = GoogleBigQuery.Database([BillingProject = #"Parameter - Source"]),\n#"gcp-project" = Source{[Name=#"Parameter - Source"]}[Data],\nuniversal_Schema = #"gcp-project"{[Name="universal",Kind="Schema"]}[Data],\nD_WH_DATE_Table = universal_Schema{[Name="D_WH_DATE",Kind="Table"]}[Data],\n#"Filtered Rows" = Table.SelectRows(D_WH_DATE_Table, each [D_DATE] > #datetime(2019, 9, 10, 0, 0, 0)),\n#"Filtered Rows1" = Table.SelectRows(#"Filtered Rows", each DateTime.IsInPreviousNHours([D_DATE], 87600))\n in \n#"Filtered Rows1"',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "snowflake native-query-with-join",
"source": [
{
"expression": 'let\n Source = Value.NativeQuery(Snowflake.Databases("xaa48144.snowflakecomputing.com","GSL_TEST_WH",[Role="ACCOUNTADMIN"]){[Name="GSL_TEST_DB"]}[Data], "select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, \'mo\')", null, [EnableFolding=true])\nin\n Source',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "job-history",
"source": [
{
"expression": 'let\n Source = Oracle.Database("localhost:1521/salesdb.domain.com", [HierarchicalNavigation=true]), HR = Source{[Schema="HR"]}[Data], EMPLOYEES1 = HR{[Name="EMPLOYEES"]}[Data] \n in EMPLOYEES1',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "postgres_test_table",
"source": [
{
"expression": 'let\n Source = PostgreSQL.Database("localhost" , "mics" ),\n public_order_date = Source{[Schema="public",Item="order_date"]}[Data] \n in \n public_order_date',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
],
},
{
"id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
"name": "hr_pbi_test",
"tables": [
{
"name": "dbo_book_issue",
"source": [
{
"expression": 'let\n Source = Sql.Database("localhost", "library"),\n dbo_book_issue = Source{[Schema="dbo",Item="book_issue"]}[Data]\n in dbo_book_issue',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
{
"name": "ms_sql_native_table",
"source": [
{
"expression": 'let\n Source = Sql.Database("AUPRDWHDB", "COMMOPSDB", [Query="select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR,\'-\',\'\'))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH,\'-\',\'\'))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION", CommandTimeout=#duration(0, 1, 30, 0)]),\n #"Changed Type" = Table.TransformColumnTypes(Source,{{"mth_date", type date}}),\n #"Added Custom" = Table.AddColumn(#"Changed Type", "Month", each Date.Month([mth_date])),\n #"Added Custom1" = Table.AddColumn(#"Added Custom", "TPV Opening", each if [Month] = 1 then [TPV_AMV_OPENING]\nelse if [Month] = 2 then 0\nelse if [Month] = 3 then 0\nelse if [Month] = 4 then [TPV_AMV_OPENING]\nelse if [Month] = 5 then 0\nelse if [Month] = 6 then 0\nelse if [Month] = 7 then [TPV_AMV_OPENING]\nelse if [Month] = 8 then 0\nelse if [Month] = 9 then 0\nelse if [Month] = 10 then [TPV_AMV_OPENING]\nelse if [Month] = 11 then 0\nelse if [Month] = 12 then 0\n\nelse 0)\nin\n #"Added Custom1"',
}
],
"datasourceUsages": [
{
"datasourceInstanceId": "DCE90B40-84D6-467A-9A5C-648E830E72D3",
}
],
},
],
},
{
"id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
"tables": [
{
"name": "employee_ctc",
"source": [
{
"expression": "dummy",
}
],
}
],
},
],
"dashboards": [
{
"id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
"isReadOnly": True,
}
],
"reports": [
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
}
],
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/a674efd1-603c-4129-8d82-03cf2be05aff": {
"method": "GET",
"status_code": 200,
"json": {
"workspaces": [
{
"id": "64ED5CAD-7C22-4684-8180-826122881108",
"name": "second-demo-workspace",
"state": "Active",
"datasets": [
{
"id": "05169CD2-E713-41E6-96AA-1D8066D95445",
"tables": [
{
"name": "public articles",
"source": [
{
"expression": "dummy",
}
],
}
],
}
],
"dashboards": [
{
"id": "7D668CAD-8FFC-4505-9215-655BCA5BEBAE",
"isReadOnly": True,
}
],
"reports": [
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
}
],
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": {
"method": "POST",
"status_code": 200,
"json": scan_init_response,
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
"webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
"embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48",
}
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715": {
"method": "GET",
"status_code": 200,
"json": {
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
"webUrl": "https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715",
"embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48",
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"displayName": "Regional Sales Analysis",
"name": "ReportSection",
"order": "0",
},
{
"displayName": "Geographic Analysis",
"name": "ReportSection1",
"order": "1",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/parameters": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"name": "Parameter - Source",
"type": "Text",
"isRequired": True,
"currentValue": "my-test-project",
},
{
"name": "My bq project",
"type": "Text",
"isRequired": True,
"currentValue": "gcp_billing",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff": {
"method": "GET",
"status_code": 200,
"json": {
"id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
"name": "employee-dataset",
"description": "Employee Management",
"webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
},
},
}
api_vs_response.update(override_data)
for url in api_vs_response.keys():
request_mock.register_uri(
api_vs_response[url]["method"],
url,
json=api_vs_response[url]["json"],
status_code=api_vs_response[url]["status_code"],
)
def default_source_config():
return {
"client_id": "foo",
"client_secret": "bar",
"tenant_id": "0B0C960B-FCDF-4D0F-8C45-2E03BB59DDEB",
"workspace_id": "64ED5CAD-7C10-4684-8180-826122881108",
"extract_lineage": False,
"extract_reports": False,
"extract_ownership": True,
"convert_lineage_urns_to_lowercase": False,
"workspace_id_pattern": {"allow": ["64ED5CAD-7C10-4684-8180-826122881108"]},
"dataset_type_mapping": {
"PostgreSql": "postgres",
"Oracle": "oracle",
},
"env": "DEV",
"extract_workspaces_to_containers": False,
"enable_advance_lineage_sql_construct": False,
}
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_powerbi_ingest(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_ingest.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_powerbi_ingest_patch_disabled(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"patch_metadata": False,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_ingest_patch_disabled.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_powerbi_test_connection_success(mock_msal):
report = test_connection_helpers.run_test_connection(
PowerBiDashboardSource, default_source_config()
)
test_connection_helpers.assert_basic_connectivity_success(report)
@freeze_time(FROZEN_TIME)
@pytest.mark.integration
def test_powerbi_test_connection_failure():
report = test_connection_helpers.run_test_connection(
PowerBiDashboardSource, default_source_config()
)
test_connection_helpers.assert_basic_connectivity_failure(
report, "Unable to get authority configuration"
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_powerbi_platform_instance_ingest(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
output_path: str = f"{tmp_path}/powerbi_platform_instance_mces.json"
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"platform_instance": "aws-ap-south-1",
},
},
"sink": {
"type": "file",
"config": {
"filename": output_path,
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_platform_instance_ingest.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=output_path,
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_powerbi_ingest_urn_lower_case(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"env": "PROD",
"platform_instance": "myPlatformInstance",
"convert_urns_to_lowercase": True,
"convert_lineage_urns_to_lowercase": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_lower_case_urn_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_lower_case_urn_ingest.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_lower_case_urn_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_override_ownership(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_ownership": False,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_mces_disabled_ownership.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
mce_out_file = "golden_test_disabled_ownership.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_mces_disabled_ownership.json",
golden_path=f"{test_resources_dir}/{mce_out_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_scan_all_workspaces(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_reports": False,
"extract_ownership": False,
"workspace_id_pattern": {
"deny": ["64ED5CAD-7322-4684-8180-826122881108"],
},
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_mces_scan_all_workspaces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_scan_all_workspaces.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_mces_scan_all_workspaces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_extract_reports(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_reports": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_report_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_report.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_report_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_extract_lineage(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-lineage-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_lineage": True,
"dataset_type_mapping": {
"PostgreSql": {"platform_instance": "operational_instance"},
"Oracle": {
"platform_instance": "high_performance_production_unit"
},
"Sql": {"platform_instance": "reporting-db"},
"Snowflake": {"platform_instance": "sn-2"},
},
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_lineage_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_lineage.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_lineage_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_extract_endorsements(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_reports": False,
"extract_endorsements_to_tags": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_endorsement_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
mce_out_file = "golden_test_endorsement.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_endorsement_mces.json",
golden_path=f"{test_resources_dir}/{mce_out_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_admin_access_is_not_allowed(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(
request_mock=requests_mock,
override_data={
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo": {
"method": "POST",
"status_code": 403,
"json": {},
},
},
)
pipeline = Pipeline.create(
{
"run_id": "powerbi-admin-api-disabled-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_lineage": True,
"dataset_type_mapping": {
"PostgreSql": {"platform_instance": "operational_instance"},
"Oracle": {
"platform_instance": "high_performance_production_unit"
},
"Sql": {"platform_instance": "reporting-db"},
"Snowflake": {"platform_instance": "sn-2"},
},
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/golden_test_admin_access_not_allowed_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_admin_access_not_allowed.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/golden_test_admin_access_not_allowed_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_workspace_container(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"workspace_id_pattern": {
"deny": ["64ED5CAD-7322-4684-8180-826122881108"],
},
"extract_workspaces_to_containers": True,
"extract_datasets_to_containers": True,
"extract_reports": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_container_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
mce_out_file = "golden_test_container.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_container_mces.json",
golden_path=f"{test_resources_dir}/{mce_out_file}",
)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_access_token_expiry_with_long_expiry(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_access_token_mces.json",
},
},
}
)
# for long expiry, the token should only be requested once.
MsalClient.token = {
"access_token": "dummy2",
"expires_in": 3600,
}
MsalClient.reset()
pipeline.run()
# We expect the token to be requested twice (once for AdminApiResolver and one for RegularApiResolver)
assert MsalClient.call_num == 2
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_access_token_expiry_with_short_expiry(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
enable_logging()
register_mock_api(request_mock=requests_mock)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_access_token_mces.json",
},
},
}
)
# for short expiry, the token should be requested when expires.
MsalClient.token = {
"access_token": "dummy",
"expires_in": 0,
}
pipeline.run()
assert MsalClient.call_num > 2
def dataset_type_mapping_set_to_all_platform(pipeline: Pipeline) -> None:
source_config: PowerBiDashboardSourceConfig = cast(
PowerBiDashboardSource, pipeline.source
).source_config
assert source_config.dataset_type_mapping is not None
# Generate default dataset_type_mapping and compare it with source_config.dataset_type_mapping
default_dataset_type_mapping: dict = {}
for item in SupportedDataPlatform:
default_dataset_type_mapping[
item.value.powerbi_data_platform_name
] = item.value.datahub_data_platform_name
assert default_dataset_type_mapping == source_config.dataset_type_mapping
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_dataset_type_mapping_should_set_to_all(
mock_msal, pytestconfig, tmp_path, mock_time, requests_mock
):
"""
Here we don't need to run the pipeline. We need to verify dataset_type_mapping is set to default dataplatform
"""
register_mock_api(request_mock=requests_mock)
new_config: dict = {**default_source_config()}
del new_config["dataset_type_mapping"]
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**new_config,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_lower_case_urn_mces.json",
},
},
}
)
dataset_type_mapping_set_to_all_platform(pipeline)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_dataset_type_mapping_error(
mock_msal, pytestconfig, tmp_path, mock_time, requests_mock
):
"""
Here we don't need to run the pipeline. We need to verify if both dataset_type_mapping and server_to_platform_instance
are set then value error should get raised
"""
register_mock_api(request_mock=requests_mock)
with pytest.raises(Exception, match=r"dataset_type_mapping is deprecated"):
Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"server_to_platform_instance": {
"localhost": {
"platform_instance": "test",
}
},
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_lower_case_urn_mces.json",
},
},
}
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_server_to_platform_map(
mock_msal, pytestconfig, tmp_path, mock_time, requests_mock
):
enable_logging()
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
new_config: dict = {
**default_source_config(),
"extract_lineage": True,
"convert_lineage_urns_to_lowercase": True,
}
del new_config["dataset_type_mapping"]
new_config["server_to_platform_instance"] = {
"hp123rt5.ap-southeast-2.fakecomputing.com": {
"platform_instance": "snowflake_production_instance",
"env": "PROD",
},
"my-test-project": {
"platform_instance": "bigquery-computing-dev-account",
"env": "QA",
},
"localhost:1521": {"platform_instance": "oracle-sales-instance", "env": "PROD"},
}
register_mock_api(request_mock=requests_mock)
output_path: str = f"{tmp_path}/powerbi_server_to_platform_instance_mces.json"
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": new_config,
},
"sink": {
"type": "file",
"config": {
"filename": output_path,
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file_path: str = (
f"{test_resources_dir}/golden_test_server_to_platform_instance.json"
)
mce_helpers.check_golden_file(
pytestconfig,
output_path=output_path,
golden_path=golden_file_path,
)
# As server_to_platform_instance map is provided, the old dataset_type_mapping
# should be set to all supported platform
# to process all available upstream lineage even if mapping for platform instance is
# not provided in server_to_platform_instance map
dataset_type_mapping_set_to_all_platform(pipeline)
def validate_pipeline(pipeline: Pipeline) -> None:
mock_workspace: Workspace = Workspace(
id="64ED5CAD-7C10-4684-8180-826122881108",
name="demo-workspace",
datasets={},
dashboards=[],
reports=[],
report_endorsements={},
dashboard_endorsements={},
scan_result={},
independent_datasets=[],
)
# Fetch actual reports
reports: List[Report] = cast(
PowerBiDashboardSource, pipeline.source
).powerbi_client.get_reports(workspace=mock_workspace)
assert len(reports) == 2
# Generate expected reports using mock reports
mock_reports: List[Dict] = [
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
"pages": [
{
"name": "ReportSection",
"displayName": "Regional Sales Analysis",
"order": "0",
},
{
"name": "ReportSection1",
"displayName": "Geographic Analysis",
"order": "1",
},
],
},
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
"name": "Product",
"description": "Acryl product report",
"pages": [],
},
]
expected_reports: List[Report] = [
Report(
id=report[Constant.ID],
name=report[Constant.NAME],
webUrl="",
embedUrl="",
description=report[Constant.DESCRIPTION],
pages=[
Page(
id="{}.{}".format(
report[Constant.ID], page[Constant.NAME].replace(" ", "_")
),
name=page[Constant.NAME],
displayName=page[Constant.DISPLAY_NAME],
order=page[Constant.ORDER],
)
for page in report["pages"]
],
users=[],
tags=[],
dataset=mock_workspace.datasets.get(report[Constant.DATASET_ID]),
)
for report in mock_reports
]
# Compare actual and expected reports
for i in range(2):
assert reports[i].id == expected_reports[i].id
assert reports[i].name == expected_reports[i].name
assert reports[i].description == expected_reports[i].description
assert reports[i].dataset == expected_reports[i].dataset
assert reports[i].pages == expected_reports[i].pages
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
@pytest.mark.integration
def test_reports_with_failed_page_request(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
"""
Test that all reports are fetched even if a single page request fails
"""
register_mock_api(
request_mock=requests_mock,
override_data={
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
"webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715",
"embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=64ED5CAD-7C10-4684-8180-826122881108",
},
{
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
"name": "Product",
"description": "Acryl product report",
"webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
"embedUrl": "https://app.powerbi.com/reportEmbed?reportId=e9fd6b0b-d8c8-4265-8c44-67e183aebf97&groupId=64ED5CAD-7C10-4684-8180-826122881108",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715": {
"method": "GET",
"status_code": 200,
"json": {
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "5b218778-e7a5-4d73-8187-f10824047715",
"name": "SalesMarketing",
"description": "Acryl sales marketing report",
"webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715",
"embedUrl": "https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=64ED5CAD-7C10-4684-8180-826122881108",
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97": {
"method": "GET",
"status_code": 200,
"json": {
"datasetId": "05169CD2-E713-41E6-9600-1D8066D95445",
"id": "e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
"name": "Product",
"description": "Acryl product report",
"webUrl": "https://app.powerbi.com/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97",
"embedUrl": "https://app.powerbi.com/reportEmbed?reportId=e9fd6b0b-d8c8-4265-8c44-67e183aebf97&groupId=64ED5CAD-7C10-4684-8180-826122881108",
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages": {
"method": "GET",
"status_code": 200,
"json": {
"value": [
{
"displayName": "Regional Sales Analysis",
"name": "ReportSection",
"order": "0",
},
{
"displayName": "Geographic Analysis",
"name": "ReportSection1",
"order": "1",
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/e9fd6b0b-d8c8-4265-8c44-67e183aebf97/pages": {
"method": "GET",
"status_code": 400,
"json": {
"error": {
"code": "InvalidRequest",
"message": "Request is currently not supported for RDL reports",
}
},
},
},
)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_reports": True,
"platform_instance": "aws-ap-south-1",
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}powerbi_reports_with_failed_page_request_mces.json",
},
},
}
)
validate_pipeline(pipeline)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_independent_datasets_extraction(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(
request_mock=requests_mock,
override_data={
"https://api.powerbi.com/v1.0/myorg/groups": {
"method": "GET",
"status_code": 200,
"json": {
"@odata.count": 3,
"value": [
{
"id": "64ED5CAD-7C10-4684-8180-826122881108",
"isReadOnly": True,
"name": "demo-workspace",
"type": "Workspace",
},
],
},
},
"https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff": {
"method": "GET",
"status_code": 200,
"json": {
"workspaces": [
{
"id": "64ED5CAD-7C10-4684-8180-826122881108",
"name": "demo-workspace",
"state": "Active",
"datasets": [
{
"id": "91580e0e-1680-4b1c-bbf9-4f6764d7a5ff",
"tables": [
{
"name": "employee_ctc",
"source": [
{
"expression": "dummy",
}
],
}
],
},
],
},
]
},
},
"https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards": {
"method": "GET",
"status_code": 200,
"json": {"value": []},
},
},
)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_source_config(),
"extract_independent_datasets": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_independent_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_independent_datasets.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_independent_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_cll_extraction(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
test_resources_dir = pytestconfig.rootpath / "tests/integration/powerbi"
register_mock_api(
request_mock=requests_mock,
)
default_conf: dict = default_source_config()
del default_conf[
"dataset_type_mapping"
] # delete this key so that connector set it to default (all dataplatform)
pipeline = Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_conf,
"extract_lineage": True,
"extract_column_level_lineage": True,
"enable_advance_lineage_sql_construct": True,
"native_query_parsing": True,
"extract_independent_datasets": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_cll_mces.json",
},
},
}
)
pipeline.run()
pipeline.raise_from_status()
golden_file = "golden_test_cll.json"
mce_helpers.check_golden_file(
pytestconfig,
output_path=f"{tmp_path}/powerbi_cll_mces.json",
golden_path=f"{test_resources_dir}/{golden_file}",
)
@freeze_time(FROZEN_TIME)
@mock.patch("msal.ConfidentialClientApplication", side_effect=mock_msal_cca)
def test_cll_extraction_flags(
mock_msal: MagicMock,
pytestconfig: pytest.Config,
tmp_path: str,
mock_time: datetime.datetime,
requests_mock: Any,
) -> None:
register_mock_api(
request_mock=requests_mock,
)
default_conf: dict = default_source_config()
pattern: str = re.escape(
"Enable all these flags in recipe: ['native_query_parsing', 'enable_advance_lineage_sql_construct', 'extract_lineage']"
)
with pytest.raises(Exception, match=pattern):
Pipeline.create(
{
"run_id": "powerbi-test",
"source": {
"type": "powerbi",
"config": {
**default_conf,
"extract_column_level_lineage": True,
},
},
"sink": {
"type": "file",
"config": {
"filename": f"{tmp_path}/powerbi_cll_mces.json",
},
},
}
)
Reference in New Issue View Git Blame Copy Permalink