2023-01-03 21:38:11 +05:30
from typing import Any , Dict
2022-03-01 07:07:22 +05:30
from unittest import mock
from freezegun import freeze_time
from datahub . ingestion . run . pipeline import Pipeline
from tests . test_helpers import mce_helpers
FROZEN_TIME = " 2022-02-03 07:00:00 "
2022-08-08 23:56:00 +05:30
def mock_msal_cca ( * args , * * kwargs ) :
class MsalClient :
def acquire_token_for_client ( self , * args , * * kwargs ) :
return {
" access_token " : " dummy " ,
}
2022-03-01 07:07:22 +05:30
2022-08-08 23:56:00 +05:30
return MsalClient ( )
2022-03-01 07:07:22 +05:30
2022-08-08 23:56:00 +05:30
2023-01-03 21:38:11 +05:30
def scan_init_response ( request , context ) :
# Request mock is passing POST input in the form of workspaces=<workspace_id>
workspace_id = request . text . split ( " = " ) [ 1 ]
w_id_vs_response : Dict [ str , Any ] = {
" 64ED5CAD-7C10-4684-8180-826122881108 " : {
" id " : " 4674efd1-603c-4129-8d82-03cf2be05aff "
} ,
" 64ED5CAD-7C22-4684-8180-826122881108 " : {
" id " : " a674efd1-603c-4129-8d82-03cf2be05aff "
} ,
}
return w_id_vs_response [ workspace_id ]
2022-11-28 18:17:15 +02:00
2022-08-08 23:56:00 +05:30
def register_mock_api ( request_mock ) :
api_vs_response = {
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/groups " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" id " : " 64ED5CAD-7C10-4684-8180-826122881108 " ,
" isReadOnly " : True ,
" name " : " Workspace 1 " ,
" type " : " Workspace " ,
} ,
{
" id " : " 64ED5CAD-7C22-4684-8180-826122881108 " ,
" isReadOnly " : True ,
" name " : " Workspace 2 " ,
" type " : " Workspace " ,
} ,
{
" id " : " 64ED5CAD-7322-4684-8180-826122881108 " ,
" isReadOnly " : True ,
" name " : " Workspace 2 " ,
" type " : " Workspace " ,
} ,
]
} ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" id " : " 7D668CAD-7FFC-4505-9215-655BCA5BEBAE " ,
" isReadOnly " : True ,
" displayName " : " test_dashboard " ,
" embedUrl " : " https://localhost/dashboards/embed/1 " ,
" webUrl " : " https://localhost/dashboards/web/1 " ,
}
]
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" id " : " 7D668CAD-8FFC-4505-9215-655BCA5BEBAE " ,
" isReadOnly " : True ,
" displayName " : " test_dashboard2 " ,
" embedUrl " : " https://localhost/dashboards/embed/1 " ,
" webUrl " : " https://localhost/dashboards/web/1 " ,
}
]
} ,
} ,
2022-11-17 12:22:23 +05:30
" https://api.powerbi.com/v1.0/myorg/admin/reports/5b218778-e7a5-4d73-8187-f10824047715/users " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" identifier " : " User1@foo.com " ,
" displayName " : " user1 " ,
" emailAddress " : " User1@foo.com " ,
" datasetUserAccessRight " : " ReadWrite " ,
" graphId " : " C9EE53F2-88EA-4711-A173-AF0515A3CD46 " ,
" principalType " : " User " ,
} ,
{
" identifier " : " User2@foo.com " ,
" displayName " : " user2 " ,
" emailAddress " : " User2@foo.com " ,
" datasetUserAccessRight " : " ReadWrite " ,
" graphId " : " C9EE53F2-88EA-4711-A173-AF0515A5REWS " ,
" principalType " : " User " ,
} ,
]
} ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/users " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" identifier " : " User1@foo.com " ,
" displayName " : " user1 " ,
" emailAddress " : " User1@foo.com " ,
" datasetUserAccessRight " : " ReadWrite " ,
" graphId " : " C9EE53F2-88EA-4711-A173-AF0515A3CD46 " ,
" principalType " : " User " ,
} ,
{
" identifier " : " User2@foo.com " ,
" displayName " : " user2 " ,
" emailAddress " : " User2@foo.com " ,
" datasetUserAccessRight " : " ReadWrite " ,
" graphId " : " C9EE53F2-88EA-4711-A173-AF0515A5REWS " ,
" principalType " : " User " ,
} ,
]
} ,
} ,
2023-01-03 21:38:11 +05:30
" https://api.powerbi.com/v1.0/myorg/admin/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/users " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" identifier " : " User3@foo.com " ,
" displayName " : " user3 " ,
" emailAddress " : " User3@foo.com " ,
" datasetUserAccessRight " : " ReadWrite " ,
" graphId " : " C9EE53F2-88EA-4711-A173-AF0515A3CD46 " ,
" principalType " : " User " ,
} ,
{
" identifier " : " User4@foo.com " ,
" displayName " : " user4 " ,
" emailAddress " : " User4@foo.com " ,
" datasetUserAccessRight " : " ReadWrite " ,
" graphId " : " C9EE53F2-88EA-4711-A173-AF0515A5REWS " ,
" principalType " : " User " ,
} ,
]
} ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/dashboards/7D668CAD-7FFC-4505-9215-655BCA5BEBAE/tiles " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" id " : " B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0 " ,
" title " : " test_tile " ,
" embedUrl " : " https://localhost/tiles/embed/1 " ,
" datasetId " : " 05169CD2-E713-41E6-9600-1D8066D95445 " ,
} ,
2023-01-03 21:38:11 +05:30
{
" id " : " 23212598-23b5-4980-87cc-5fc0ecd84385 " ,
" title " : " yearly_sales " ,
" embedUrl " : " https://localhost/tiles/embed/2 " ,
" datasetId " : " ba0130a1-5b03-40de-9535-b34e778ea6ed " ,
} ,
2022-08-08 23:56:00 +05:30
]
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/dashboards/7D668CAD-8FFC-4505-9215-655BCA5BEBAE/tiles " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : { " value " : [ ] } ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445 " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" id " : " 05169CD2-E713-41E6-9600-1D8066D95445 " ,
" name " : " library-dataset " ,
" webUrl " : " http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445 " ,
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445 " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" id " : " 05169CD2-E713-41E6-96AA-1D8066D95445 " ,
" name " : " library-dataset " ,
" webUrl " : " http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445 " ,
} ,
} ,
2023-01-03 21:38:11 +05:30
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" id " : " ba0130a1-5b03-40de-9535-b34e778ea6ed " ,
" name " : " hr_pbi_test " ,
" webUrl " : " http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed " ,
} ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445/datasources " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" datasourceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
" datasourceType " : " PostgreSql " ,
" connectionDetails " : {
" database " : " library_db " ,
" server " : " foo " ,
2022-03-01 07:07:22 +05:30
} ,
} ,
2022-08-08 23:56:00 +05:30
]
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445/datasources " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" datasourceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
" datasourceType " : " PostgreSql " ,
" connectionDetails " : {
" database " : " library_db " ,
" server " : " foo " ,
} ,
} ,
]
} ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/4674efd1-603c-4129-8d82-03cf2be05aff " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" status " : " SUCCEEDED " ,
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanStatus/a674efd1-603c-4129-8d82-03cf2be05aff " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" status " : " SUCCEEDED " ,
} ,
} ,
2022-08-08 23:56:00 +05:30
" https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/4674efd1-603c-4129-8d82-03cf2be05aff " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" workspaces " : [
{
" id " : " 64ED5CAD-7C10-4684-8180-826122881108 " ,
" name " : " demo-workspace " ,
" state " : " Active " ,
" datasets " : [
{
" id " : " 05169CD2-E713-41E6-9600-1D8066D95445 " ,
2023-01-03 21:38:11 +05:30
" name " : " test_sf_pbi_test " ,
2022-08-08 23:56:00 +05:30
" tables " : [
{
" name " : " public issue_history " ,
" source " : [
{
" expression " : " dummy " ,
}
] ,
2023-01-03 21:38:11 +05:30
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
{
" name " : " SNOWFLAKE_TESTTABLE " ,
" source " : [
{
" expression " : ' let \n Source = Snowflake.Databases( " hp123rt5.ap-southeast-2.fakecomputing.com " , " PBI_TEST_WAREHOUSE_PROD " ,[Role= " PBI_TEST_MEMBER " ]), \n PBI_TEST_Database = Source { [Name= " PBI_TEST " ,Kind= " Database " ]}[Data], \n TEST_Schema = PBI_TEST_Database { [Name= " TEST " ,Kind= " Schema " ]}[Data], \n TESTTABLE_Table = TEST_Schema { [Name= " TESTTABLE " ,Kind= " Table " ]}[Data] \n in \n TESTTABLE_Table ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
{
" name " : " snowflake native-query " ,
" source " : [
{
" expression " : ' let \n Source = Value.NativeQuery(Snowflake.Databases( " bu20658.ap-southeast-2.snowflakecomputing.com " , " operations_analytics_warehouse_prod " ,[Role= " OPERATIONS_ANALYTICS_MEMBER " ]) { [Name= " OPERATIONS_ANALYTICS " ]}[Data], " SELECT#(lf)concat((UPPER(REPLACE(SELLER, \' - \' , \' \' ))), MONTHID) as AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR, \' - \' , \' \' ))), MONTHID) as CD_AGENT_KEY,#(lf) *#(lf)FROM#(lf)OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_APS_SME_UNITS_V4 " , null, [EnableFolding=true]), \n # " Added Conditional Column " = Table.AddColumn(Source, " SME Units ENT " , each if [DEAL_TYPE] = " SME Unit " then [UNIT] else 0), \n # " Added Conditional Column1 " = Table.AddColumn(# " Added Conditional Column " , " Banklink Units " , each if [DEAL_TYPE] = " Banklink " then [UNIT] else 0), \n # " Removed Columns " = Table.RemoveColumns(# " Added Conditional Column1 " , { " Banklink Units " }), \n # " Added Custom " = Table.AddColumn(# " Removed Columns " , " Banklink Units " , each if [DEAL_TYPE] = " Banklink " and [SALES_TYPE] = " 3 - Upsell " \n then [UNIT] \n \n else if [SALES_TYPE] = " Adjusted BL Migration " \n then [UNIT] \n \n else 0), \n # " Added Custom1 " = Table.AddColumn(# " Added Custom " , " SME Units in $ (*$361) " , each if [DEAL_TYPE] = " SME Unit " \n and [SALES_TYPE] <> " 4 - Renewal " \n then [UNIT] * 361 \n else 0), \n # " Added Custom2 " = Table.AddColumn(# " Added Custom1 " , " Banklink in $ (*$148) " , each [Banklink Units] * 148) \n in \n # " Added Custom2 " ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
{
" name " : " snowflake native-query-with-join " ,
" source " : [
{
" expression " : ' let \n Source = Value.NativeQuery(Snowflake.Databases( " xaa48144.snowflakecomputing.com " , " GSL_TEST_WH " ,[Role= " ACCOUNTADMIN " ]) { [Name= " GSL_TEST_DB " ]}[Data], " select A.name from GSL_TEST_DB.PUBLIC.SALES_ANALYST as A inner join GSL_TEST_DB.PUBLIC.SALES_FORECAST as B on A.name = B.name where startswith(A.name, \' mo \' ) " , null, [EnableFolding=true]) \n in \n Source ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
{
" name " : " job-history " ,
" source " : [
{
" expression " : ' let \n Source = Oracle.Database( " localhost:1521/salesdb.GSLAB.COM " , [HierarchicalNavigation=true]), HR = Source { [Schema= " HR " ]}[Data], EMPLOYEES1 = HR { [Name= " EMPLOYEES " ]}[Data] \n in EMPLOYEES1 ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
{
" name " : " postgres_test_table " ,
" source " : [
{
" expression " : ' let \n Source = PostgreSQL.Database( " localhost " , " mics " ), \n public_order_date = Source { [Schema= " public " ,Item= " order_date " ]}[Data] \n in \n public_order_date ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
2022-08-08 23:56:00 +05:30
] ,
2023-01-03 21:38:11 +05:30
} ,
{
" id " : " ba0130a1-5b03-40de-9535-b34e778ea6ed " ,
" name " : " hr_pbi_test " ,
" tables " : [
{
" name " : " dbo_book_issue " ,
" source " : [
{
" expression " : ' let \n Source = Sql.Database( " localhost " , " library " ), \n dbo_book_issue = Source { [Schema= " dbo " ,Item= " book_issue " ]}[Data] \n in dbo_book_issue ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
{
" name " : " ms_sql_native_table " ,
" source " : [
{
" expression " : ' let \n Source = Sql.Database( " AUPRDWHDB " , " COMMOPSDB " , [Query= " select *,#(lf)concat((UPPER(REPLACE(CLIENT_DIRECTOR, \' - \' , \' \' ))), MONTH_WID) as CD_AGENT_KEY,#(lf)concat((UPPER(REPLACE(CLIENT_MANAGER_CLOSING_MONTH, \' - \' , \' \' ))), MONTH_WID) as AGENT_KEY#(lf)#(lf)from V_PS_CD_RETENTION " , CommandTimeout=#duration(0, 1, 30, 0)]), \n # " Changed Type " = Table.TransformColumnTypes(Source, {{ " mth_date " , type date}}), \n # " Added Custom " = Table.AddColumn(# " Changed Type " , " Month " , each Date.Month([mth_date])), \n # " Added Custom1 " = Table.AddColumn(# " Added Custom " , " TPV Opening " , each if [Month] = 1 then [TPV_AMV_OPENING] \n else if [Month] = 2 then 0 \n else if [Month] = 3 then 0 \n else if [Month] = 4 then [TPV_AMV_OPENING] \n else if [Month] = 5 then 0 \n else if [Month] = 6 then 0 \n else if [Month] = 7 then [TPV_AMV_OPENING] \n else if [Month] = 8 then 0 \n else if [Month] = 9 then 0 \n else if [Month] = 10 then [TPV_AMV_OPENING] \n else if [Month] = 11 then 0 \n else if [Month] = 12 then 0 \n \n else 0) \n in \n # " Added Custom1 " ' ,
}
] ,
" datasourceUsages " : [
{
" datasourceInstanceId " : " DCE90B40-84D6-467A-9A5C-648E830E72D3 " ,
}
] ,
} ,
] ,
} ,
2022-08-08 23:56:00 +05:30
] ,
2022-03-01 07:07:22 +05:30
} ,
2022-08-08 23:56:00 +05:30
]
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/admin/workspaces/scanResult/a674efd1-603c-4129-8d82-03cf2be05aff " : {
" method " : " GET " ,
2022-08-08 23:56:00 +05:30
" status_code " : 200 ,
" json " : {
2022-11-28 18:17:15 +02:00
" workspaces " : [
{
" id " : " 64ED5CAD-7C22-4684-8180-826122881108 " ,
" name " : " second-demo-workspace " ,
" state " : " Active " ,
" datasets " : [
{
" id " : " 05169CD2-E713-41E6-96AA-1D8066D95445 " ,
" tables " : [
{
" name " : " public articles " ,
" source " : [
{
" expression " : " dummy " ,
}
] ,
}
] ,
}
] ,
} ,
]
2022-08-08 23:56:00 +05:30
} ,
} ,
2022-11-28 18:17:15 +02:00
" https://api.powerbi.com/v1.0/myorg/admin/workspaces/getInfo " : {
" method " : " POST " ,
" status_code " : 200 ,
" json " : scan_init_response ,
} ,
2022-11-17 12:22:23 +05:30
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" datasetId " : " 05169CD2-E713-41E6-9600-1D8066D95445 " ,
" id " : " 5b218778-e7a5-4d73-8187-f10824047715 " ,
" name " : " SalesMarketing " ,
" description " : " Acryl sales marketing report " ,
" webUrl " : " https://app.powerbi.com/groups/f089354e-8366-4e18-aea3-4cb4a3a50b48/reports/5b218778-e7a5-4d73-8187-f10824047715 " ,
" embedUrl " : " https://app.powerbi.com/reportEmbed?reportId=5b218778-e7a5-4d73-8187-f10824047715&groupId=f089354e-8366-4e18-aea3-4cb4a3a50b48 " ,
}
]
} ,
} ,
" https://api.powerbi.com/v1.0/myorg/groups/64ED5CAD-7C10-4684-8180-826122881108/reports/5b218778-e7a5-4d73-8187-f10824047715/pages " : {
" method " : " GET " ,
" status_code " : 200 ,
" json " : {
" value " : [
{
" displayName " : " Regional Sales Analysis " ,
" name " : " ReportSection " ,
" order " : " 0 " ,
} ,
{
" displayName " : " Geographic Analysis " ,
" name " : " ReportSection1 " ,
" order " : " 1 " ,
} ,
]
} ,
} ,
2022-08-08 23:56:00 +05:30
}
for url in api_vs_response . keys ( ) :
request_mock . register_uri (
api_vs_response [ url ] [ " method " ] ,
url ,
json = api_vs_response [ url ] [ " json " ] ,
status_code = api_vs_response [ url ] [ " status_code " ] ,
2022-03-01 07:07:22 +05:30
)
2022-08-08 23:56:00 +05:30
def default_source_config ( ) :
return {
" client_id " : " foo " ,
" client_secret " : " bar " ,
" tenant_id " : " 0B0C960B-FCDF-4D0F-8C45-2E03BB59DDEB " ,
2023-01-03 21:38:11 +05:30
" workspace_id " : " 64ED5CAD-7C10-4684-8180-826122881108 " ,
" extract_lineage " : False ,
" extract_reports " : False ,
" convert_lineage_urns_to_lowercase " : False ,
2022-11-28 18:17:15 +02:00
" workspace_id_pattern " : { " allow " : [ " 64ED5CAD-7C10-4684-8180-826122881108 " ] } ,
2022-08-08 23:56:00 +05:30
" dataset_type_mapping " : {
" PostgreSql " : " postgres " ,
" Oracle " : " oracle " ,
} ,
" env " : " DEV " ,
}
@freeze_time ( FROZEN_TIME )
@mock.patch ( " msal.ConfidentialClientApplication " , side_effect = mock_msal_cca )
def test_powerbi_ingest ( mock_msal , pytestconfig , tmp_path , mock_time , requests_mock ) :
2022-11-28 18:17:15 +02:00
2022-08-08 23:56:00 +05:30
test_resources_dir = pytestconfig . rootpath / " tests/integration/powerbi "
register_mock_api ( request_mock = requests_mock )
pipeline = Pipeline . create (
{
" run_id " : " powerbi-test " ,
" source " : {
" type " : " powerbi " ,
" config " : {
* * default_source_config ( ) ,
} ,
} ,
" sink " : {
" type " : " file " ,
" config " : {
" filename " : f " { tmp_path } /powerbi_mces.json " ,
} ,
} ,
}
)
pipeline . run ( )
pipeline . raise_from_status ( )
2023-01-03 21:38:11 +05:30
golden_file = " golden_test_ingest.json "
2022-08-08 23:56:00 +05:30
mce_helpers . check_golden_file (
pytestconfig ,
output_path = tmp_path / " powerbi_mces.json " ,
2023-01-03 21:38:11 +05:30
golden_path = f " { test_resources_dir } / { golden_file } " ,
)
@freeze_time ( FROZEN_TIME )
@mock.patch ( " msal.ConfidentialClientApplication " , side_effect = mock_msal_cca )
def test_powerbi_ingest_urn_lower_case (
mock_msal , pytestconfig , tmp_path , mock_time , requests_mock
) :
test_resources_dir = pytestconfig . rootpath / " tests/integration/powerbi "
register_mock_api ( request_mock = requests_mock )
pipeline = Pipeline . create (
{
" run_id " : " powerbi-test " ,
" source " : {
" type " : " powerbi " ,
" config " : {
* * default_source_config ( ) ,
" convert_urns_to_lowercase " : True ,
" convert_lineage_urns_to_lowercase " : True ,
} ,
} ,
" sink " : {
" type " : " file " ,
" config " : {
" filename " : f " { tmp_path } /powerbi_lower_case_urn_mces.json " ,
} ,
} ,
}
)
pipeline . run ( )
pipeline . raise_from_status ( )
golden_file = " golden_test_lower_case_urn_ingest.json "
mce_helpers . check_golden_file (
pytestconfig ,
output_path = f " { tmp_path } /powerbi_lower_case_urn_mces.json " ,
golden_path = f " { test_resources_dir } / { golden_file } " ,
2022-08-08 23:56:00 +05:30
)
@freeze_time ( FROZEN_TIME )
@mock.patch ( " msal.ConfidentialClientApplication " , side_effect = mock_msal_cca )
def test_override_ownership (
mock_msal , pytestconfig , tmp_path , mock_time , requests_mock
) :
2022-11-28 18:17:15 +02:00
2022-08-08 23:56:00 +05:30
test_resources_dir = pytestconfig . rootpath / " tests/integration/powerbi "
register_mock_api ( request_mock = requests_mock )
pipeline = Pipeline . create (
{
" run_id " : " powerbi-test " ,
" source " : {
" type " : " powerbi " ,
" config " : {
* * default_source_config ( ) ,
" extract_ownership " : False ,
} ,
} ,
" sink " : {
" type " : " file " ,
" config " : {
" filename " : f " { tmp_path } /powerbi_mces_disabled_ownership.json " ,
} ,
} ,
}
)
pipeline . run ( )
pipeline . raise_from_status ( )
mce_out_file = " golden_test_disabled_ownership.json "
mce_helpers . check_golden_file (
pytestconfig ,
output_path = tmp_path / " powerbi_mces_disabled_ownership.json " ,
golden_path = f " { test_resources_dir } / { mce_out_file } " ,
)
2022-11-17 12:22:23 +05:30
2022-11-28 18:17:15 +02:00
@freeze_time ( FROZEN_TIME )
@mock.patch ( " msal.ConfidentialClientApplication " , side_effect = mock_msal_cca )
def test_scan_all_workspaces (
mock_msal , pytestconfig , tmp_path , mock_time , requests_mock
) :
test_resources_dir = pytestconfig . rootpath / " tests/integration/powerbi "
register_mock_api ( request_mock = requests_mock )
pipeline = Pipeline . create (
{
" run_id " : " powerbi-test " ,
" source " : {
" type " : " powerbi " ,
" config " : {
* * default_source_config ( ) ,
" extract_reports " : False ,
" extract_ownership " : False ,
" workspace_id_pattern " : {
" deny " : [ " 64ED5CAD-7322-4684-8180-826122881108 " ] ,
} ,
} ,
} ,
" sink " : {
" type " : " file " ,
" config " : {
" filename " : f " { tmp_path } /powerbi_mces_scan_all_workspaces.json " ,
} ,
} ,
}
)
pipeline . run ( )
pipeline . raise_from_status ( )
2023-01-03 21:38:11 +05:30
golden_file = " golden_test_scan_all_workspaces.json "
2022-11-28 18:17:15 +02:00
mce_helpers . check_golden_file (
pytestconfig ,
output_path = tmp_path / " powerbi_mces_scan_all_workspaces.json " ,
2023-01-03 21:38:11 +05:30
golden_path = f " { test_resources_dir } / { golden_file } " ,
2022-11-28 18:17:15 +02:00
)
2022-11-17 12:22:23 +05:30
@freeze_time ( FROZEN_TIME )
@mock.patch ( " msal.ConfidentialClientApplication " , side_effect = mock_msal_cca )
def test_extract_reports ( mock_msal , pytestconfig , tmp_path , mock_time , requests_mock ) :
2022-11-28 18:17:15 +02:00
2022-11-17 12:22:23 +05:30
test_resources_dir = pytestconfig . rootpath / " tests/integration/powerbi "
register_mock_api ( request_mock = requests_mock )
pipeline = Pipeline . create (
{
" run_id " : " powerbi-test " ,
" source " : {
" type " : " powerbi " ,
" config " : {
* * default_source_config ( ) ,
2023-01-03 21:38:11 +05:30
" extract_reports " : True ,
2022-11-17 12:22:23 +05:30
} ,
} ,
" sink " : {
" type " : " file " ,
" config " : {
" filename " : f " { tmp_path } /powerbi_report_mces.json " ,
} ,
} ,
}
)
pipeline . run ( )
pipeline . raise_from_status ( )
2023-01-03 21:38:11 +05:30
golden_file = " golden_test_report.json "
2022-11-17 12:22:23 +05:30
mce_helpers . check_golden_file (
pytestconfig ,
output_path = tmp_path / " powerbi_report_mces.json " ,
2023-01-03 21:38:11 +05:30
golden_path = f " { test_resources_dir } / { golden_file } " ,
)
@freeze_time ( FROZEN_TIME )
@mock.patch ( " msal.ConfidentialClientApplication " , side_effect = mock_msal_cca )
def test_extract_lineage ( mock_msal , pytestconfig , tmp_path , mock_time , requests_mock ) :
test_resources_dir = pytestconfig . rootpath / " tests/integration/powerbi "
register_mock_api ( request_mock = requests_mock )
pipeline = Pipeline . create (
{
" run_id " : " powerbi-lineage-test " ,
" source " : {
" type " : " powerbi " ,
" config " : {
* * default_source_config ( ) ,
" extract_lineage " : True ,
" dataset_type_mapping " : {
" PostgreSql " : { " platform_instance " : " operational_instance " } ,
" Oracle " : {
" platform_instance " : " high_performance_production_unit "
} ,
" Sql " : { " platform_instance " : " reporting-db " } ,
" Snowflake " : { " platform_instance " : " sn-2 " } ,
} ,
} ,
} ,
" sink " : {
" type " : " file " ,
" config " : {
" filename " : f " { tmp_path } /powerbi_lineage_mces.json " ,
} ,
} ,
}
)
pipeline . run ( )
pipeline . raise_from_status ( )
golden_file = " golden_test_lineage.json "
mce_helpers . check_golden_file (
pytestconfig ,
output_path = f " { tmp_path } /powerbi_lineage_mces.json " ,
golden_path = f " { test_resources_dir } / { golden_file } " ,
2022-11-17 12:22:23 +05:30
)
