yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-10 18:26:24 +00:00
Code Issues Packages Projects Releases Wiki Activity
datahub/assets/js/c9fe24a5.94e29315.js
chriscollins3456 cbb4b7f488 deploy: 5685eda1200042cd47bb0c3d2c44d4b6ef7a51bb
2025-08-08 21:37:06 +00:00

1 line
79 KiB
JavaScript
Raw Blame History

"use strict";(self.webpackChunkdocs_website=self.webpackChunkdocs_website||[]).push([[39088],{13368:(e,a,t)=>{t.r(a),t.d(a,{assets:()=>m,contentTitle:()=>p,default:()=>f,frontMatter:()=>d,metadata:()=>g,toc:()=>c});t(96540);var n=t(15680),i=t(53720),s=t(5400);function l(e,a,t){return a in e?Object.defineProperty(e,a,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[a]=t,e}function r(e,a){return a=null!=a?a:{},Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(a)):function(e,a){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var n=Object.getOwnPropertySymbols(e);a&&(n=n.filter((function(a){return Object.getOwnPropertyDescriptor(e,a).enumerable}))),t.push.apply(t,n)}return t}(Object(a)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(a,t))})),e}function o(e,a){if(null==e)return{};var t,n,i=function(e,a){if(null==e)return{};var t,n,i={},s=Object.keys(e);for(n=0;n<s.length;n++)t=s[n],a.indexOf(t)>=0||(i[t]=e[t]);return i}(e,a);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);for(n=0;n<s.length;n++)t=s[n],a.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(i[t]=e[t])}return i}const d={sidebar_position:51,title:"PowerBI",slug:"/generated/ingestion/sources/powerbi",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/powerbi.md"},p="PowerBI",g={unversionedId:"docs/generated/ingestion/sources/powerbi",id:"docs/generated/ingestion/sources/powerbi",title:"PowerBI",description:"Certified",source:"@site/genDocs/docs/generated/ingestion/sources/powerbi.md",sourceDirName:"docs/generated/ingestion/sources",slug:"/generated/ingestion/sources/powerbi",permalink:"/docs/generated/ingestion/sources/powerbi",draft:!1,editUrl:"https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/powerbi.md",tags:[],version:"current",sidebarPosition:51,frontMatter:{sidebar_position:51,title:"PowerBI",slug:"/generated/ingestion/sources/powerbi",custom_edit_url:"https://github.com/datahub-project/datahub/blob/master/docs/generated/ingestion/sources/powerbi.md"},sidebar:"overviewSidebar",previous:{title:"Postgres",permalink:"/docs/generated/ingestion/sources/postgres"},next:{title:"PowerBI Report Server",permalink:"/docs/generated/ingestion/sources/powerbi-report-server"}},m={},c=[{value:"Important Capabilities",id:"important-capabilities",level:3},{value:"Configuration Notes",id:"configuration-notes",level:2},{value:"Concept mapping",id:"concept-mapping",level:2},{value:"Lineage",id:"lineage",level:2},{value:"M-Query Pattern Supported For Lineage Extraction",id:"m-query-pattern-supported-for-lineage-extraction",level:2},{value:"Extract endorsements to tags",id:"extract-endorsements-to-tags",level:2},{value:"Profiling",id:"profiling",level:2},{value:"Admin Ingestion vs. Basic Ingestion",id:"admin-ingestion-vs-basic-ingestion",level:2},{value:"Admin Ingestion: Service Principal As Admin in Tenant Setting and Added as Member In Workspace",id:"admin-ingestion-service-principal-as-admin-in-tenant-setting-and-added-as-member-in-workspace",level:3},{value:"Basic Ingestion: Service Principal As Member In Workspace",id:"basic-ingestion-service-principal-as-member-in-workspace",level:3},{value:"CLI based Ingestion",id:"cli-based-ingestion",level:3},{value:"Starter Recipe",id:"starter-recipe",level:3},{value:"Config Details",id:"config-details",level:3},{value:"Code Coordinates",id:"code-coordinates",level:3}],y={toc:c},u="wrapper";function f(e){var{components:a}=e,t=o(e,["components"]);return(0,n.yg)(u,r(function(e){for(var a=1;a<arguments.length;a++){var t=null!=arguments[a]?arguments[a]:{},n=Object.keys(t);"function"==typeof Object.getOwnPropertySymbols&&(n=n.concat(Object.getOwnPropertySymbols(t).filter((function(e){return Object.getOwnPropertyDescriptor(t,e).enumerable})))),n.forEach((function(a){l(e,a,t[a])}))}return e}({},y,t),{components:a,mdxType:"MDXLayout"}),(0,n.yg)("h1",{id:"powerbi"},"PowerBI"),(0,n.yg)("p",null,(0,n.yg)("img",{parentName:"p",src:"https://img.shields.io/badge/support%20status-certified-brightgreen",alt:"Certified"})),(0,n.yg)("h3",{id:"important-capabilities"},"Important Capabilities"),(0,n.yg)("table",null,(0,n.yg)("thead",{parentName:"table"},(0,n.yg)("tr",{parentName:"thead"},(0,n.yg)("th",{parentName:"tr",align:null},"Capability"),(0,n.yg)("th",{parentName:"tr",align:null},"Status"),(0,n.yg)("th",{parentName:"tr",align:null},"Notes"))),(0,n.yg)("tbody",{parentName:"table"},(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Asset Containers"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Column-level Lineage"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Disabled by default, configured using ",(0,n.yg)("inlineCode",{parentName:"td"},"extract_column_level_lineage"),". .")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("a",{parentName:"td",href:"/docs/metadata-ingestion/docs/dev_guides/sql_profiles"},"Data Profiling")),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Optionally enabled via configuration profiling.enabled.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Descriptions"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("a",{parentName:"td",href:"/docs/metadata-ingestion/docs/dev_guides/stateful#stale-entity-removal"},"Detect Deleted Entities")),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default via stateful ingestion.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Extract Ownership"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Extract Tags"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("a",{parentName:"td",href:"/docs/platform-instances"},"Platform Instance")),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Schema Metadata"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Table-Level Lineage"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default, configured using ",(0,n.yg)("inlineCode",{parentName:"td"},"extract_lineage"),".")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},"Test Connection"),(0,n.yg)("td",{parentName:"tr",align:null},"\u2705"),(0,n.yg)("td",{parentName:"tr",align:null},"Enabled by default.")))),(0,n.yg)("p",null,"This plugin extracts the following:"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Power BI dashboards, tiles and datasets"),(0,n.yg)("li",{parentName:"ul"},"Names, descriptions and URLs of dashboard and tile"),(0,n.yg)("li",{parentName:"ul"},"Owners of dashboards")),(0,n.yg)("h2",{id:"configuration-notes"},"Configuration Notes"),(0,n.yg)("ol",null,(0,n.yg)("li",{parentName:"ol"},(0,n.yg)("p",{parentName:"li"},"Refer ",(0,n.yg)("a",{parentName:"p",href:"https://docs.microsoft.com/en-us/power-bi/developer/embedded/embed-service-principal"},"Microsoft AD App Creation doc")," to create a Microsoft AD Application. Once Microsoft AD Application is created you can configure client-credential i.e. client_id and client_secret in recipe for ingestion.")),(0,n.yg)("li",{parentName:"ol"},(0,n.yg)("p",{parentName:"li"},"Enable admin access if you want to ingest data source and dataset information, including lineage, and endorsement tags. Refer section ",(0,n.yg)("a",{parentName:"p",href:"#admin-ingestion-vs-basic-ingestion"},"Admin Ingestion vs. Basic Ingestion")," for more detail."),(0,n.yg)("p",{parentName:"li"},"Login to PowerBI as Admin and from ",(0,n.yg)("inlineCode",{parentName:"p"},"Admin API settings")," allow below permissions"),(0,n.yg)("ul",{parentName:"li"},(0,n.yg)("li",{parentName:"ul"},"Allow service principals to use read-only admin APIs"),(0,n.yg)("li",{parentName:"ul"},"Enhance admin APIs responses with detailed metadata"),(0,n.yg)("li",{parentName:"ul"},"Enhance admin APIs responses with DAX and mashup expressions")))),(0,n.yg)("h2",{id:"concept-mapping"},"Concept mapping"),(0,n.yg)("table",null,(0,n.yg)("thead",{parentName:"table"},(0,n.yg)("tr",{parentName:"thead"},(0,n.yg)("th",{parentName:"tr",align:null},"PowerBI"),(0,n.yg)("th",{parentName:"tr",align:null},"Datahub"))),(0,n.yg)("tbody",{parentName:"table"},(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dashboard")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dashboard"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dataset's Table")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dataset"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Tile")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Chart"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Report.webUrl")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Chart.externalUrl"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Workspace")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Container"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Report")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dashboard"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"PaginatedReport")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dashboard"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Page")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Chart"))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"App")),(0,n.yg)("td",{parentName:"tr",align:null},(0,n.yg)("inlineCode",{parentName:"td"},"Dashboard"))))),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"If ",(0,n.yg)("inlineCode",{parentName:"li"},"Tile")," is created from report then ",(0,n.yg)("inlineCode",{parentName:"li"},"Chart.externalUrl")," is set to Report.webUrl."),(0,n.yg)("li",{parentName:"ul"},"The ",(0,n.yg)("inlineCode",{parentName:"li"},"Page")," is unavailable for PowerBI PaginatedReport.")),(0,n.yg)("h2",{id:"lineage"},"Lineage"),(0,n.yg)("p",null,"This source extracts table lineage for tables present in PowerBI Datasets. Lets consider a PowerBI Dataset ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_REPORT")," and a PostgreSQL database is configured as data-source in ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_REPORT")," dataset."),(0,n.yg)("p",null,"Consider ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_REPORT")," PowerBI Dataset has a table ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_ANALYSIS")," which is backed by ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_ANALYSIS_VIEW")," of PostgreSQL Database then in this case ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_ANALYSIS_VIEW")," will appear as upstream dataset for ",(0,n.yg)("inlineCode",{parentName:"p"},"SALES_ANALYSIS")," table."),(0,n.yg)("p",null,"You can control table lineage ingestion using ",(0,n.yg)("inlineCode",{parentName:"p"},"extract_lineage")," configuration parameter, by default it is set to ",(0,n.yg)("inlineCode",{parentName:"p"},"true"),"."),(0,n.yg)("p",null,"PowerBI Source extracts the lineage information by parsing PowerBI M-Query expressions and from dataset data returned by the PowerBI API."),(0,n.yg)("p",null,"The source will attempt to extract information from ODBC connection strings in M-Query expressions to determine the database type. If the database type matches a supported platform and the source is able to extract enough information to construct a valid Dataset URN, it will extract lineage for that data source."),(0,n.yg)("p",null,"PowerBI Source will extract lineage for the below listed PowerBI Data Sources:"),(0,n.yg)("ol",null,(0,n.yg)("li",{parentName:"ol"},"Snowflake"),(0,n.yg)("li",{parentName:"ol"},"Oracle"),(0,n.yg)("li",{parentName:"ol"},"PostgreSQL"),(0,n.yg)("li",{parentName:"ol"},"Microsoft SQL Server"),(0,n.yg)("li",{parentName:"ol"},"Google BigQuery"),(0,n.yg)("li",{parentName:"ol"},"Databricks"),(0,n.yg)("li",{parentName:"ol"},"MySQL")),(0,n.yg)("p",null,"Native SQL query parsing is supported for ",(0,n.yg)("inlineCode",{parentName:"p"},"Snowflake")," and ",(0,n.yg)("inlineCode",{parentName:"p"},"Amazon Redshift")," data sources."),(0,n.yg)("p",null,"For example, consider the SQL query shown below. The table ",(0,n.yg)("inlineCode",{parentName:"p"},"OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_UNIT_TARGET")," will be ingested as an upstream table."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},'let\n Source = Value.NativeQuery(\n Snowflake.Databases(\n "sdfsd788.ws-east-2.fakecomputing.com",\n "operations_analytics_prod",\n [Role = "OPERATIONS_ANALYTICS_MEMBER"]\n ){[Name = "OPERATIONS_ANALYTICS"]}[Data],\n "select #(lf)UPPER(REPLACE(AGENT_NAME,\\\'-\\\',\\\'\\\')) AS Agent,#(lf)TIER,#(lf)UPPER(MANAGER),#(lf)TEAM_TYPE,#(lf)DATE_TARGET,#(lf)MONTHID,#(lf)TARGET_TEAM,#(lf)SELLER_EMAIL,#(lf)concat((UPPER(REPLACE(AGENT_NAME,\\\'-\\\',\\\'\\\'))), MONTHID) as AGENT_KEY,#(lf)UNIT_TARGET AS SME_Quota,#(lf)AMV_TARGET AS Revenue_Quota,#(lf)SERVICE_QUOTA,#(lf)BL_TARGET,#(lf)SOFTWARE_QUOTA as Software_Quota#(lf)#(lf)from OPERATIONS_ANALYTICS.TRANSFORMED_PROD.V_UNIT_TARGETS#(lf)#(lf)where YEAR_TARGET >= 2020#(lf)and TEAM_TYPE = \\\'foo\\\'#(lf)and TARGET_TEAM = \\\'bar\\\'",\n null,\n [EnableFolding = true]\n ),\n #"Added Conditional Column" = Table.AddColumn(\n Source,\n "Has PS Software Quota?",\n each\n if [TIER] = "Expansion (Medium)" then\n "Yes"\n else if [TIER] = "Acquisition" then\n "Yes"\n else\n "No"\n )\nin\n #"Added Conditional Column"\n')),(0,n.yg)("p",null,"Use full-table-name in ",(0,n.yg)("inlineCode",{parentName:"p"},"from")," clause. For example dev.public.category"),(0,n.yg)("h2",{id:"m-query-pattern-supported-for-lineage-extraction"},"M-Query Pattern Supported For Lineage Extraction"),(0,n.yg)("p",null,"Lets consider a M-Query which combine two PostgreSQL tables. Such M-Query can be written as per below patterns."),(0,n.yg)("p",null,(0,n.yg)("strong",{parentName:"p"},"Pattern-1")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},'let\nSource = PostgreSQL.Database("localhost", "book_store"),\nbook_date = Source{[Schema="public",Item="book"]}[Data],\nissue_history = Source{[Schema="public",Item="issue_history"]}[Data],\ncombine_result = Table.Combine({book_date, issue_history})\nin\ncombine_result\n')),(0,n.yg)("p",null,(0,n.yg)("strong",{parentName:"p"},"Pattern-2")),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-shell"},'let\nSource = PostgreSQL.Database("localhost", "book_store"),\ncombine_result = Table.Combine({Source{[Schema="public",Item="book"]}[Data], Source{[Schema="public",Item="issue_history"]}[Data]})\nin\ncombine_result\n')),(0,n.yg)("p",null,(0,n.yg)("inlineCode",{parentName:"p"},"Pattern-2")," is ",(0,n.yg)("em",{parentName:"p"},"not")," supported for upstream table lineage extraction as it uses nested item-selector i.e. {Source{",'[Schema="public",Item="book"]',"}","[Data]",", Source{",'[Schema="public",Item="issue_history"]',"}","[Data]","} as argument to M-QUery table function i.e. Table.Combine"),(0,n.yg)("p",null,(0,n.yg)("inlineCode",{parentName:"p"},"Pattern-1")," is supported as it first assigns the table from schema to variable and then variable is used in M-Query Table function i.e. Table.Combine"),(0,n.yg)("h2",{id:"extract-endorsements-to-tags"},"Extract endorsements to tags"),(0,n.yg)("p",null,"By default, extracting endorsement information to tags is disabled. The feature may be useful if organization uses ",(0,n.yg)("a",{parentName:"p",href:"https://learn.microsoft.com/en-us/power-bi/collaborate-share/service-endorse-content"},"endorsements")," to identify content quality."),(0,n.yg)("p",null,"Please note that the default implementation overwrites tags for the ingested entities, if you need to preserve existing tags, consider using a ",(0,n.yg)("a",{parentName:"p",href:"/docs/metadata-ingestion/docs/transformer/dataset_transformer#simple-add-dataset-globaltags"},"transformer")," with ",(0,n.yg)("inlineCode",{parentName:"p"},"semantics: PATCH")," tags instead of ",(0,n.yg)("inlineCode",{parentName:"p"},"OVERWRITE"),"."),(0,n.yg)("h2",{id:"profiling"},"Profiling"),(0,n.yg)("p",null,"The profiling implementation is done through querying ",(0,n.yg)("a",{parentName:"p",href:"https://learn.microsoft.com/en-us/rest/api/power-bi/datasets/execute-queries"},"DAX query endpoint"),". Therefore, the principal needs to have permission to query the datasets to be profiled. Usually this means that the service principal should have ",(0,n.yg)("inlineCode",{parentName:"p"},"Contributor")," role for the workspace to be ingested. Profiling is done with column-based queries to be able to handle wide datasets without timeouts."),(0,n.yg)("p",null,"Take into account that the profiling implementation executes a fairly big number of DAX queries, and for big datasets this is a significant load to the PowerBI system."),(0,n.yg)("p",null,"The ",(0,n.yg)("inlineCode",{parentName:"p"},"profiling_pattern")," setting may be used to limit profiling actions to only a certain set of resources in PowerBI. Both allowed and deny rules are matched against the following pattern for every table in a PowerBI Dataset: ",(0,n.yg)("inlineCode",{parentName:"p"},"workspace_name.dataset_name.table_name"),". Users may limit profiling with these settings at table level, dataset level or workspace level."),(0,n.yg)("h2",{id:"admin-ingestion-vs-basic-ingestion"},"Admin Ingestion vs. Basic Ingestion"),(0,n.yg)("p",null,"PowerBI provides two sets of API i.e. ",(0,n.yg)("a",{parentName:"p",href:"https://learn.microsoft.com/en-us/rest/api/power-bi/"},"Basic API and Admin API"),"."),(0,n.yg)("p",null,"The Basic API returns metadata of PowerBI resources where service principal has granted access explicitly on resources,\nwhereas Admin API returns metadata of all PowerBI resources irrespective of whether service principal has granted\nor doesn't grant access explicitly on resources."),(0,n.yg)("p",null,"The Admin Ingestion (explained below) is the recommended way to execute PowerBI ingestion as this ingestion can extract most of the metadata."),(0,n.yg)("h3",{id:"admin-ingestion-service-principal-as-admin-in-tenant-setting-and-added-as-member-in-workspace"},"Admin Ingestion: Service Principal As Admin in Tenant Setting and Added as Member In Workspace"),(0,n.yg)("p",null,"To grant admin access to the service principal, visit your PowerBI tenant Settings."),(0,n.yg)("p",null,"If you have added service principal as ",(0,n.yg)("inlineCode",{parentName:"p"},"member")," in workspace and also allowed below permissions from PowerBI tenant Settings"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Allow service principal to use read-only PowerBI Admin APIs"),(0,n.yg)("li",{parentName:"ul"},"Enhance admin APIs responses with detailed metadata"),(0,n.yg)("li",{parentName:"ul"},"Enhance admin APIs responses with DAX and mashup expressions")),(0,n.yg)("p",null,"PowerBI Source would be able to ingest below listed metadata of that particular workspace"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Lineage"),(0,n.yg)("li",{parentName:"ul"},"PowerBI Dataset"),(0,n.yg)("li",{parentName:"ul"},"Endorsement as tag"),(0,n.yg)("li",{parentName:"ul"},"Dashboards"),(0,n.yg)("li",{parentName:"ul"},"Reports"),(0,n.yg)("li",{parentName:"ul"},"Dashboard Tiles"),(0,n.yg)("li",{parentName:"ul"},"Report Pages"),(0,n.yg)("li",{parentName:"ul"},"App")),(0,n.yg)("p",null,"If you don't want to add a service principal as a member in your workspace, then you can enable the ",(0,n.yg)("inlineCode",{parentName:"p"},"admin_apis_only: true")," in recipe to use PowerBI Admin API only."),(0,n.yg)("p",null,"Caveats of setting ",(0,n.yg)("inlineCode",{parentName:"p"},"admin_apis_only")," to ",(0,n.yg)("inlineCode",{parentName:"p"},"true"),":"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Report's pages would not get ingested as page API is not available in PowerBI Admin API"),(0,n.yg)("li",{parentName:"ul"},(0,n.yg)("a",{parentName:"li",href:"https://learn.microsoft.com/en-us/power-query/power-query-query-parameters"},"PowerBI Parameters")," would not get resolved to actual values while processing M-Query for table lineage"),(0,n.yg)("li",{parentName:"ul"},"Dataset profiling is unavailable, as it requires access to the workspace API")),(0,n.yg)("h3",{id:"basic-ingestion-service-principal-as-member-in-workspace"},"Basic Ingestion: Service Principal As Member In Workspace"),(0,n.yg)("p",null,"If you have added service principal as ",(0,n.yg)("inlineCode",{parentName:"p"},"member")," in workspace then PowerBI Source would be able to ingest below metadata of that particular workspace"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Dashboards"),(0,n.yg)("li",{parentName:"ul"},"Reports"),(0,n.yg)("li",{parentName:"ul"},"Dashboard's Tiles"),(0,n.yg)("li",{parentName:"ul"},"Report's Pages")),(0,n.yg)("h3",{id:"cli-based-ingestion"},"CLI based Ingestion"),(0,n.yg)("h3",{id:"starter-recipe"},"Starter Recipe"),(0,n.yg)("p",null,"Check out the following recipe to get started with ingestion! See ",(0,n.yg)("a",{parentName:"p",href:"#config-details"},"below")," for full configuration options."),(0,n.yg)("p",null,"For general pointers on writing and running a recipe, see our ",(0,n.yg)("a",{parentName:"p",href:"/docs/metadata-ingestion#recipes"},"main recipe guide"),"."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-yaml"},'source:\n type: "powerbi"\n config:\n # Your Power BI tenant identifier\n tenant_id: a949d688-67c0-4bf1-a344-e939411c6c0a\n\n # Azure AD Application identifier\n client_id: foo\n # Azure AD App client secret\n client_secret: bar\n \n # Ingest elements of below PowerBi Workspace into Datahub\n workspace_name_pattern:\n allow:\n - MyWorkspace\n deny:\n - PrivateWorkspace\n\n # Enable / Disable ingestion of ownership information for dashboards\n extract_ownership: true\n \n # Enable/Disable extracting workspace information to DataHub containers\n extract_workspaces_to_containers: true\n \n # Enable / Disable ingestion of endorsements.\n # Please notice that this may overwrite any existing tags defined to ingested entities!\n extract_endorsements_to_tags: false\n \n # Optional -- This mapping is optional and only required to configure platform-instance for upstream tables\n # A mapping of PowerBI datasource\'s server i.e host[:port] to data platform instance.\n # :port is optional and only needed if your datasource server is running on non-standard port.\n # For Google BigQuery the datasource\'s server is google bigquery project name\n server_to_platform_instance:\n ap-south-1.snowflakecomputing.com:\n platform_instance: operational_instance\n env: DEV\n oracle-server:1920:\n platform_instance: high_performance_production_unit\n env: PROD\n big-query-sales-project:\n platform_instance: sn-2\n env: QA\n\n # Need admin_api, only ingest workspace that are modified since...\n modified_since: "2023-02-10T00:00:00.0000000Z"\n\n ownership:\n # create powerbi user as datahub corpuser, false will still extract ownership of workspace/ dashboards\n create_corp_user: false\n # use email to build user urn instead of powerbi user identifier\n use_powerbi_email: true\n # remove email suffix like @acryl.io\n remove_email_suffix: true\n # only ingest user with certain authority\n owner_criteria: ["ReadWriteReshareExplore","Owner","Admin"]\n # wrap powerbi tables (datahub dataset) under 1 powerbi dataset (datahub container)\n extract_datasets_to_containers: true\n # only ingest dataset that are endorsed, like "Certified"\n filter_dataset_endorsements: \n allow:\n - Certified\n \n # extract powerbi dashboards and tiles\n extract_dashboards: false\n # extract powerbi dataset table schema\n extract_dataset_schema: true\n\n # Enable PowerBI dataset profiling\n profiling:\n enabled: false\n # Pattern to limit which resources to profile\n # Matched resource format is following:\n # workspace_name.dataset_name.table_name\n profile_pattern:\n deny:\n - .*\n\n\nsink:\n # sink configs\n\n')),(0,n.yg)("h3",{id:"config-details"},"Config Details"),(0,n.yg)(i.A,{mdxType:"Tabs"},(0,n.yg)(s.A,{value:"options",label:"Options",default:!0,mdxType:"TabItem"},(0,n.yg)("p",null,"Note that a ",(0,n.yg)("inlineCode",{parentName:"p"},".")," is used to denote nested fields in the YAML recipe."),(0,n.yg)("div",{className:"config-table"},(0,n.yg)("table",null,(0,n.yg)("thead",{parentName:"table"},(0,n.yg)("tr",{parentName:"thead"},(0,n.yg)("th",{parentName:"tr",align:"left"},"Field"),(0,n.yg)("th",{parentName:"tr",align:"left"},"Description"))),(0,n.yg)("tbody",{parentName:"table"},(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"client_id"),"\xa0",(0,n.yg)("abbr",{title:"Required"},"\u2705"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Azure app client identifier")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"client_secret"),"\xa0",(0,n.yg)("abbr",{title:"Required"},"\u2705"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Azure app client secret")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"tenant_id"),"\xa0",(0,n.yg)("abbr",{title:"Required"},"\u2705"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"PowerBI tenant identifier")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"admin_apis_only"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Retrieve metadata using PowerBI Admin API only. If this is enabled, then Report Pages will not be extracted. Admin API access is required if this setting is enabled ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"convert_lineage_urns_to_lowercase"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to convert the urns of ingested lineage dataset to lowercase ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"convert_urns_to_lowercase"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to convert the PowerBI assets urns to lowercase ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"dsn_to_platform_name"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"map(str,string)"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"enable_advance_lineage_sql_construct"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to enable advance native sql construct for parsing like join, sub-queries. along this flag , the native_query_parsing should be enabled. By default convert_lineage_urns_to_lowercase is enabled, in-case if you have disabled it in previous ingestion execution then it may break lineageas this option generates the upstream datasets URN in lowercase. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_app"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to ingest workspace app. Requires DataHub server 0.14.2+. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_column_level_lineage"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to extract column level lineage. Works only if configs ",(0,n.yg)("inlineCode",{parentName:"td"},"native_query_parsing"),", ",(0,n.yg)("inlineCode",{parentName:"td"},"enable_advance_lineage_sql_construct")," & ",(0,n.yg)("inlineCode",{parentName:"td"},"extract_lineage")," are enabled.Works for M-Query where native SQL is used for transformation. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_dashboards"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to ingest PBI Dashboard and Tiles as Datahub Dashboard and Chart ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_dataset_schema"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to ingest PBI Dataset Table columns and measures. Note: this setting must be ",(0,n.yg)("inlineCode",{parentName:"td"},"true")," for schema extraction and column lineage to be enabled. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_datasets_to_containers"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"PBI tables will be grouped under a Datahub Container, the container reflect a PBI Dataset ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_endorsements_to_tags"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to extract endorsements to tags, note that this may overwrite existing tags. Admin API access is required if this setting is enabled. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_independent_datasets"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to extract datasets not used in any PowerBI visualization ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_lineage"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether lineage should be ingested between X and Y. Admin API access is required if this setting is enabled ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_ownership"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether ownership should be ingested. Admin API access is required if this setting is enabled. Note that enabling this may overwrite owners that you've added inside DataHub's web application. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_reports"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether reports should be ingested ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"extract_workspaces_to_containers"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Extract workspaces to DataHub containers ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"include_workspace_name_in_dataset_urn"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"It is recommended to set this to true, as it helps prevent the overwriting of datasets.Read section #11560 at ",(0,n.yg)("a",{parentName:"td",href:"https://docs.datahub.com/docs/how/updating-datahub/"},"https://docs.datahub.com/docs/how/updating-datahub/")," before enabling this option.To maintain backward compatibility, this is set to False. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"incremental_lineage"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"m_query_parse_timeout"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"integer"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Timeout for PowerBI M-query parsing in seconds. Table-level lineage is determined by analyzing the M-query expression. Increase this value if you encounter the 'M-Query Parsing Timeout' message in the connector report. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"70")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"modified_since"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Get only recently modified workspaces based on modified_since datetime '2023-02-10T00:00:00.0000000Z', excludeInActiveWorkspaces limit to last 30 days")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"native_query_parsing"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether PowerBI native query should be parsed to extract lineage ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"patch_metadata"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Patch dashboard metadata ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"platform_instance"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The instance of the platform that all assets produced by this recipe belong to")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"profile_pattern"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"AllowDenyPattern"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Regex patterns to filter tables for profiling during ingestion. Note that only tables allowed by the ",(0,n.yg)("inlineCode",{parentName:"td"},"table_pattern")," will be considered. Matched format is 'workspacename.datasetname.tablename' ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"{","'","allow","'",": ","[","'",".","*","'","]",", ","'","deny","'",": ","[","]",", ","'","ignoreCase","'",": True","}")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"scan_batch_size"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"integer"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"batch size for sending workspace_ids to PBI, 100 is the limit ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"1")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"scan_timeout"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"integer"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"timeout for PowerBI metadata scanning ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"60")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"workspace_id_as_urn_part"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"It is recommended to set this to True only if you have legacy workspaces based on Office 365 groups, as those workspaces can have identical names. To maintain backward compatibility, this is set to False which uses workspace name ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"workspace_id_pattern"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"AllowDenyPattern"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Regex patterns to filter PowerBI workspaces in ingestion by ID. By default all IDs are allowed unless they are filtered by name using 'workspace_name_pattern'. Note: This field works in conjunction with 'workspace_type_filter' and both must be considered when filtering workspaces. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"{","'","allow","'",": ","[","'",".","*","'","]",", ","'","deny","'",": ","[","]",", ","'","ignoreCase","'",": True","}")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"workspace_name_pattern"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"AllowDenyPattern"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Regex patterns to filter PowerBI workspaces in ingestion by name. By default all names are allowed unless they are filtered by ID using 'workspace_id_pattern'. Note: This field works in conjunction with 'workspace_type_filter' and both must be considered when filtering workspaces. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"{","'","allow","'",": ","[","'",".","*","'","]",", ","'","deny","'",": ","[","]",", ","'","ignoreCase","'",": True","}")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"env"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The environment that all assets produced by this connector belong to ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"PROD")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"filter_dataset_endorsements"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"AllowDenyPattern"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Filter and ingest datasets which are 'Certified' or 'Promoted' endorsement. If both are added, dataset which are 'Certified' or 'Promoted' will be ingested . Default setting allows all dataset to be ingested ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"{","'","allow","'",": ","[","'",".","*","'","]",", ","'","deny","'",": ","[","]",", ","'","ignoreCase","'",": True","}")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"filter_dataset_endorsements."),(0,n.yg)("span",{className:"path-main"},"ignoreCase"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether to ignore case sensitivity during pattern matching. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"filter_dataset_endorsements."),(0,n.yg)("span",{className:"path-main"},"allow"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"array"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"List of regex patterns to include in ingestion ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"[","'",".","*","'","]")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"filter_dataset_endorsements.allow."),(0,n.yg)("span",{className:"path-main"},"string"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"filter_dataset_endorsements."),(0,n.yg)("span",{className:"path-main"},"deny"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"array"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"List of regex patterns to exclude from ingestion. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"[","]")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"filter_dataset_endorsements.deny."),(0,n.yg)("span",{className:"path-main"},"string"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"ownership"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"OwnershipMapping"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Configure how is ownership ingested ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"{","'","create","_","corp","_","user","'",": True, ","'","use","_","powerbi","_","email","'",": Tr...")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"ownership."),(0,n.yg)("span",{className:"path-main"},"create_corp_user"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether ingest PowerBI user as Datahub Corpuser ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"ownership."),(0,n.yg)("span",{className:"path-main"},"dataset_configured_by_as_owner"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Take PBI dataset configuredBy as dataset owner if exist ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"ownership."),(0,n.yg)("span",{className:"path-main"},"remove_email_suffix"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Remove PowerBI User email suffix for example, @acryl.io ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"ownership."),(0,n.yg)("span",{className:"path-main"},"use_powerbi_email"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Use PowerBI User email to ingest as corpuser, default is powerbi user identifier ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"ownership."),(0,n.yg)("span",{className:"path-main"},"owner_criteria"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"array"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Need to have certain authority to qualify as owner for example ","['ReadWriteReshareExplore','Owner','Admin']")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"ownership.owner_criteria."),(0,n.yg)("span",{className:"path-main"},"string"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"server_to_platform_instance"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"One of map(str,union), map(str,union)"))),(0,n.yg)("td",{parentName:"tr",align:"left"})),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"server_to_platform_instance.",(0,n.yg)("inlineCode",{parentName:"td"},"key"),"."),(0,n.yg)("span",{className:"path-main"},"platform_instance"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"DataHub platform instance name. To generate correct urn for upstream dataset, this should match with platform instance name used in ingestion recipe of other datahub sources.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"server_to_platform_instance.",(0,n.yg)("inlineCode",{parentName:"td"},"key"),"."),(0,n.yg)("span",{className:"path-main"},"metastore"),"\xa0",(0,n.yg)("abbr",{title:"Required if server_to_platform_instance is set"},"\u2753"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Databricks Unity Catalog metastore name.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"server_to_platform_instance.",(0,n.yg)("inlineCode",{parentName:"td"},"key"),"."),(0,n.yg)("span",{className:"path-main"},"env"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"string"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"The environment that all assets produced by DataHub platform ingestion source belong to ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"PROD")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"workspace_type_filter"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"array"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Ingest the metadata of the workspace where the workspace type corresponds to the specified workspace_type_filter. Note: This field works in conjunction with 'workspace_id_pattern'. Both must be matched for a workspace to be processed. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"[","'","Workspace","'","]")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"workspace_type_filter."),(0,n.yg)("span",{className:"path-main"},"enum"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"Enum"))),(0,n.yg)("td",{parentName:"tr",align:"left"},'One of: "Workspace", "PersonalGroup", "Personal", "AdminWorkspace", "AdminInsights"')),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"profiling"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"PowerBiProfilingConfig"))),(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"default-line "},"Default: ",(0,n.yg)("span",{className:"default-value"},"{","'","enabled","'",": False","}")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"profiling."),(0,n.yg)("span",{className:"path-main"},"enabled"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether profiling of PowerBI datasets should be done ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-main"},"stateful_ingestion"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"StatefulStaleMetadataRemovalConfig"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"PowerBI Stateful Ingestion Config.")),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"stateful_ingestion."),(0,n.yg)("span",{className:"path-main"},"enabled"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or ",(0,n.yg)("inlineCode",{parentName:"td"},"datahub_api")," is specified, otherwise False ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"False")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"stateful_ingestion."),(0,n.yg)("span",{className:"path-main"},"fail_safe_threshold"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"number"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"75.0")))),(0,n.yg)("tr",{parentName:"tbody"},(0,n.yg)("td",{parentName:"tr",align:"left"},(0,n.yg)("div",{className:"path-line"},(0,n.yg)("span",{className:"path-prefix"},"stateful_ingestion."),(0,n.yg)("span",{className:"path-main"},"remove_stale_metadata"))," ",(0,n.yg)("div",{className:"type-name-line"},(0,n.yg)("span",{className:"type-name"},"boolean"))),(0,n.yg)("td",{parentName:"tr",align:"left"},"Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled. ",(0,n.yg)("div",{className:"default-line default-line-with-docs"},"Default: ",(0,n.yg)("span",{className:"default-value"},"True")))))))),(0,n.yg)(s.A,{value:"schema",label:"Schema",mdxType:"TabItem"},(0,n.yg)("p",null,"The ",(0,n.yg)("a",{parentName:"p",href:"https://json-schema.org/"},"JSONSchema")," for this configuration is inlined below."),(0,n.yg)("pre",null,(0,n.yg)("code",{parentName:"pre",className:"language-javascript"},'{\n "title": "PowerBiDashboardSourceConfig",\n "description": "Base configuration class for stateful ingestion for source configs to inherit from.",\n "type": "object",\n "properties": {\n "incremental_lineage": {\n "title": "Incremental Lineage",\n "description": "When enabled, emits lineage as incremental to existing lineage already in DataHub. When disabled, re-states lineage on each run.",\n "default": false,\n "type": "boolean"\n },\n "env": {\n "title": "Env",\n "description": "The environment that all assets produced by this connector belong to",\n "default": "PROD",\n "type": "string"\n },\n "platform_instance": {\n "title": "Platform Instance",\n "description": "The instance of the platform that all assets produced by this recipe belong to",\n "type": "string"\n },\n "stateful_ingestion": {\n "title": "Stateful Ingestion",\n "description": "PowerBI Stateful Ingestion Config.",\n "allOf": [\n {\n "$ref": "#/definitions/StatefulStaleMetadataRemovalConfig"\n }\n ]\n },\n "tenant_id": {\n "title": "Tenant Id",\n "description": "PowerBI tenant identifier",\n "type": "string"\n },\n "workspace_id_pattern": {\n "title": "Workspace Id Pattern",\n "description": "Regex patterns to filter PowerBI workspaces in ingestion by ID. By default all IDs are allowed unless they are filtered by name using \'workspace_name_pattern\'. Note: This field works in conjunction with \'workspace_type_filter\' and both must be considered when filtering workspaces.",\n "default": {\n "allow": [\n ".*"\n ],\n "deny": [],\n "ignoreCase": true\n },\n "allOf": [\n {\n "$ref": "#/definitions/AllowDenyPattern"\n }\n ]\n },\n "workspace_name_pattern": {\n "title": "Workspace Name Pattern",\n "description": "Regex patterns to filter PowerBI workspaces in ingestion by name. By default all names are allowed unless they are filtered by ID using \'workspace_id_pattern\'. Note: This field works in conjunction with \'workspace_type_filter\' and both must be considered when filtering workspaces.",\n "default": {\n "allow": [\n ".*"\n ],\n "deny": [],\n "ignoreCase": true\n },\n "allOf": [\n {\n "$ref": "#/definitions/AllowDenyPattern"\n }\n ]\n },\n "server_to_platform_instance": {\n "title": "Server To Platform Instance",\n "description": "A mapping of PowerBI datasource\'s server i.e host[:port] to Data platform instance. :port is optional and only needed if your datasource server is running on non-standard port. For Google BigQuery the datasource\'s server is google bigquery project name. For Databricks Unity Catalog the datasource\'s server is workspace FQDN.",\n "default": {},\n "type": "object",\n "additionalProperties": {\n "anyOf": [\n {\n "$ref": "#/definitions/PlatformDetail"\n },\n {\n "$ref": "#/definitions/DataBricksPlatformDetail"\n }\n ]\n }\n },\n "dsn_to_platform_name": {\n "title": "Dsn To Platform Name",\n "description": "A mapping of ODBC DSN to DataHub data platform name. For example with an ODBC connection string \'DSN=database\' where the database type is \'PostgreSQL\' you would configure the mapping as \'database: postgres\'.",\n "default": {},\n "type": "object",\n "additionalProperties": {\n "type": "string"\n }\n },\n "client_id": {\n "title": "Client Id",\n "description": "Azure app client identifier",\n "type": "string"\n },\n "client_secret": {\n "title": "Client Secret",\n "description": "Azure app client secret",\n "type": "string"\n },\n "scan_timeout": {\n "title": "Scan Timeout",\n "description": "timeout for PowerBI metadata scanning",\n "default": 60,\n "type": "integer"\n },\n "scan_batch_size": {\n "title": "Scan Batch Size",\n "description": "batch size for sending workspace_ids to PBI, 100 is the limit",\n "default": 1,\n "exclusiveMinimum": 0,\n "maximum": 100,\n "type": "integer"\n },\n "workspace_id_as_urn_part": {\n "title": "Workspace Id As Urn Part",\n "description": "It is recommended to set this to True only if you have legacy workspaces based on Office 365 groups, as those workspaces can have identical names. To maintain backward compatibility, this is set to False which uses workspace name",\n "default": false,\n "type": "boolean"\n },\n "extract_ownership": {\n "title": "Extract Ownership",\n "description": "Whether ownership should be ingested. Admin API access is required if this setting is enabled. Note that enabling this may overwrite owners that you\'ve added inside DataHub\'s web application.",\n "default": false,\n "type": "boolean"\n },\n "extract_reports": {\n "title": "Extract Reports",\n "description": "Whether reports should be ingested",\n "default": true,\n "type": "boolean"\n },\n "ownership": {\n "title": "Ownership",\n "description": "Configure how is ownership ingested",\n "default": {\n "create_corp_user": true,\n "use_powerbi_email": true,\n "remove_email_suffix": false,\n "dataset_configured_by_as_owner": false,\n "owner_criteria": null\n },\n "allOf": [\n {\n "$ref": "#/definitions/OwnershipMapping"\n }\n ]\n },\n "modified_since": {\n "title": "Modified Since",\n "description": "Get only recently modified workspaces based on modified_since datetime \'2023-02-10T00:00:00.0000000Z\', excludeInActiveWorkspaces limit to last 30 days",\n "type": "string"\n },\n "extract_dashboards": {\n "title": "Extract Dashboards",\n "description": "Whether to ingest PBI Dashboard and Tiles as Datahub Dashboard and Chart",\n "default": true,\n "type": "boolean"\n },\n "extract_dataset_schema": {\n "title": "Extract Dataset Schema",\n "description": "Whether to ingest PBI Dataset Table columns and measures. Note: this setting must be `true` for schema extraction and column lineage to be enabled.",\n "default": true,\n "type": "boolean"\n },\n "extract_lineage": {\n "title": "Extract Lineage",\n "description": "Whether lineage should be ingested between X and Y. Admin API access is required if this setting is enabled",\n "default": true,\n "type": "boolean"\n },\n "extract_endorsements_to_tags": {\n "title": "Extract Endorsements To Tags",\n "description": "Whether to extract endorsements to tags, note that this may overwrite existing tags. Admin API access is required if this setting is enabled.",\n "default": false,\n "type": "boolean"\n },\n "filter_dataset_endorsements": {\n "title": "Filter Dataset Endorsements",\n "description": "Filter and ingest datasets which are \'Certified\' or \'Promoted\' endorsement. If both are added, dataset which are \'Certified\' or \'Promoted\' will be ingested . Default setting allows all dataset to be ingested",\n "default": {\n "allow": [\n ".*"\n ],\n "deny": [],\n "ignoreCase": true\n },\n "allOf": [\n {\n "$ref": "#/definitions/AllowDenyPattern"\n }\n ]\n },\n "extract_workspaces_to_containers": {\n "title": "Extract Workspaces To Containers",\n "description": "Extract workspaces to DataHub containers",\n "default": true,\n "type": "boolean"\n },\n "extract_datasets_to_containers": {\n "title": "Extract Datasets To Containers",\n "description": "PBI tables will be grouped under a Datahub Container, the container reflect a PBI Dataset",\n "default": false,\n "type": "boolean"\n },\n "native_query_parsing": {\n "title": "Native Query Parsing",\n "description": "Whether PowerBI native query should be parsed to extract lineage",\n "default": true,\n "type": "boolean"\n },\n "convert_urns_to_lowercase": {\n "title": "Convert Urns To Lowercase",\n "description": "Whether to convert the PowerBI assets urns to lowercase",\n "default": false,\n "type": "boolean"\n },\n "convert_lineage_urns_to_lowercase": {\n "title": "Convert Lineage Urns To Lowercase",\n "description": "Whether to convert the urns of ingested lineage dataset to lowercase",\n "default": true,\n "type": "boolean"\n },\n "admin_apis_only": {\n "title": "Admin Apis Only",\n "description": "Retrieve metadata using PowerBI Admin API only. If this is enabled, then Report Pages will not be extracted. Admin API access is required if this setting is enabled",\n "default": false,\n "type": "boolean"\n },\n "extract_independent_datasets": {\n "title": "Extract Independent Datasets",\n "description": "Whether to extract datasets not used in any PowerBI visualization",\n "default": false,\n "type": "boolean"\n },\n "enable_advance_lineage_sql_construct": {\n "title": "Enable Advance Lineage Sql Construct",\n "description": "Whether to enable advance native sql construct for parsing like join, sub-queries. along this flag , the native_query_parsing should be enabled. By default convert_lineage_urns_to_lowercase is enabled, in-case if you have disabled it in previous ingestion execution then it may break lineageas this option generates the upstream datasets URN in lowercase.",\n "default": true,\n "type": "boolean"\n },\n "extract_column_level_lineage": {\n "title": "Extract Column Level Lineage",\n "description": "Whether to extract column level lineage. Works only if configs `native_query_parsing`, `enable_advance_lineage_sql_construct` & `extract_lineage` are enabled.Works for M-Query where native SQL is used for transformation.",\n "default": false,\n "type": "boolean"\n },\n "profile_pattern": {\n "title": "Profile Pattern",\n "description": "Regex patterns to filter tables for profiling during ingestion. Note that only tables allowed by the `table_pattern` will be considered. Matched format is \'workspacename.datasetname.tablename\'",\n "default": {\n "allow": [\n ".*"\n ],\n "deny": [],\n "ignoreCase": true\n },\n "allOf": [\n {\n "$ref": "#/definitions/AllowDenyPattern"\n }\n ]\n },\n "profiling": {\n "title": "Profiling",\n "default": {\n "enabled": false\n },\n "allOf": [\n {\n "$ref": "#/definitions/PowerBiProfilingConfig"\n }\n ]\n },\n "patch_metadata": {\n "title": "Patch Metadata",\n "description": "Patch dashboard metadata",\n "default": true,\n "type": "boolean"\n },\n "workspace_type_filter": {\n "title": "Workspace Type Filter",\n "description": "Ingest the metadata of the workspace where the workspace type corresponds to the specified workspace_type_filter. Note: This field works in conjunction with \'workspace_id_pattern\'. Both must be matched for a workspace to be processed.",\n "default": [\n "Workspace"\n ],\n "type": "array",\n "items": {\n "enum": [\n "Workspace",\n "PersonalGroup",\n "Personal",\n "AdminWorkspace",\n "AdminInsights"\n ],\n "type": "string"\n }\n },\n "include_workspace_name_in_dataset_urn": {\n "title": "Include Workspace Name In Dataset Urn",\n "description": "It is recommended to set this to true, as it helps prevent the overwriting of datasets.Read section #11560 at https://docs.datahub.com/docs/how/updating-datahub/ before enabling this option.To maintain backward compatibility, this is set to False.",\n "default": false,\n "type": "boolean"\n },\n "extract_app": {\n "title": "Extract App",\n "description": "Whether to ingest workspace app. Requires DataHub server 0.14.2+.",\n "default": false,\n "type": "boolean"\n },\n "m_query_parse_timeout": {\n "title": "M Query Parse Timeout",\n "description": "Timeout for PowerBI M-query parsing in seconds. Table-level lineage is determined by analyzing the M-query expression. Increase this value if you encounter the \'M-Query Parsing Timeout\' message in the connector report.",\n "default": 70,\n "type": "integer"\n }\n },\n "required": [\n "tenant_id",\n "client_id",\n "client_secret"\n ],\n "additionalProperties": false,\n "definitions": {\n "DynamicTypedStateProviderConfig": {\n "title": "DynamicTypedStateProviderConfig",\n "type": "object",\n "properties": {\n "type": {\n "title": "Type",\n "description": "The type of the state provider to use. For DataHub use `datahub`",\n "type": "string"\n },\n "config": {\n "title": "Config",\n "description": "The configuration required for initializing the state provider. Default: The datahub_api config if set at pipeline level. Otherwise, the default DatahubClientConfig. See the defaults (https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/graph/client.py#L19).",\n "default": {},\n "type": "object"\n }\n },\n "required": [\n "type"\n ],\n "additionalProperties": false\n },\n "StatefulStaleMetadataRemovalConfig": {\n "title": "StatefulStaleMetadataRemovalConfig",\n "description": "Base specialized config for Stateful Ingestion with stale metadata removal capability.",\n "type": "object",\n "properties": {\n "enabled": {\n "title": "Enabled",\n "description": "Whether or not to enable stateful ingest. Default: True if a pipeline_name is set and either a datahub-rest sink or `datahub_api` is specified, otherwise False",\n "default": false,\n "type": "boolean"\n },\n "remove_stale_metadata": {\n "title": "Remove Stale Metadata",\n "description": "Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.",\n "default": true,\n "type": "boolean"\n },\n "fail_safe_threshold": {\n "title": "Fail Safe Threshold",\n "description": "Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the \'fail_safe_threshold\'.",\n "default": 75.0,\n "minimum": 0.0,\n "maximum": 100.0,\n "type": "number"\n }\n },\n "additionalProperties": false\n },\n "AllowDenyPattern": {\n "title": "AllowDenyPattern",\n "description": "A class to store allow deny regexes",\n "type": "object",\n "properties": {\n "allow": {\n "title": "Allow",\n "description": "List of regex patterns to include in ingestion",\n "default": [\n ".*"\n ],\n "type": "array",\n "items": {\n "type": "string"\n }\n },\n "deny": {\n "title": "Deny",\n "description": "List of regex patterns to exclude from ingestion.",\n "default": [],\n "type": "array",\n "items": {\n "type": "string"\n }\n },\n "ignoreCase": {\n "title": "Ignorecase",\n "description": "Whether to ignore case sensitivity during pattern matching.",\n "default": true,\n "type": "boolean"\n }\n },\n "additionalProperties": false\n },\n "PlatformDetail": {\n "title": "PlatformDetail",\n "type": "object",\n "properties": {\n "platform_instance": {\n "title": "Platform Instance",\n "description": "DataHub platform instance name. To generate correct urn for upstream dataset, this should match with platform instance name used in ingestion recipe of other datahub sources.",\n "type": "string"\n },\n "env": {\n "title": "Env",\n "description": "The environment that all assets produced by DataHub platform ingestion source belong to",\n "default": "PROD",\n "type": "string"\n }\n },\n "additionalProperties": false\n },\n "DataBricksPlatformDetail": {\n "title": "DataBricksPlatformDetail",\n "description": "metastore is an additional field used in Databricks connector to generate the dataset urn",\n "type": "object",\n "properties": {\n "platform_instance": {\n "title": "Platform Instance",\n "description": "DataHub platform instance name. To generate correct urn for upstream dataset, this should match with platform instance name used in ingestion recipe of other datahub sources.",\n "type": "string"\n },\n "env": {\n "title": "Env",\n "description": "The environment that all assets produced by DataHub platform ingestion source belong to",\n "default": "PROD",\n "type": "string"\n },\n "metastore": {\n "title": "Metastore",\n "description": "Databricks Unity Catalog metastore name.",\n "type": "string"\n }\n },\n "required": [\n "metastore"\n ],\n "additionalProperties": false\n },\n "OwnershipMapping": {\n "title": "OwnershipMapping",\n "type": "object",\n "properties": {\n "create_corp_user": {\n "title": "Create Corp User",\n "description": "Whether ingest PowerBI user as Datahub Corpuser",\n "default": true,\n "type": "boolean"\n },\n "use_powerbi_email": {\n "title": "Use Powerbi Email",\n "description": "Use PowerBI User email to ingest as corpuser, default is powerbi user identifier",\n "default": true,\n "type": "boolean"\n },\n "remove_email_suffix": {\n "title": "Remove Email Suffix",\n "description": "Remove PowerBI User email suffix for example, @acryl.io",\n "default": false,\n "type": "boolean"\n },\n "dataset_configured_by_as_owner": {\n "title": "Dataset Configured By As Owner",\n "description": "Take PBI dataset configuredBy as dataset owner if exist",\n "default": false,\n "type": "boolean"\n },\n "owner_criteria": {\n "title": "Owner Criteria",\n "description": "Need to have certain authority to qualify as owner for example [\'ReadWriteReshareExplore\',\'Owner\',\'Admin\']",\n "type": "array",\n "items": {\n "type": "string"\n }\n }\n },\n "additionalProperties": false\n },\n "PowerBiProfilingConfig": {\n "title": "PowerBiProfilingConfig",\n "type": "object",\n "properties": {\n "enabled": {\n "title": "Enabled",\n "description": "Whether profiling of PowerBI datasets should be done",\n "default": false,\n "type": "boolean"\n }\n },\n "additionalProperties": false\n }\n }\n}\n')))),(0,n.yg)("h3",{id:"code-coordinates"},"Code Coordinates"),(0,n.yg)("ul",null,(0,n.yg)("li",{parentName:"ul"},"Class Name: ",(0,n.yg)("inlineCode",{parentName:"li"},"datahub.ingestion.source.powerbi.powerbi.PowerBiDashboardSource")),(0,n.yg)("li",{parentName:"ul"},"Browse on ",(0,n.yg)("a",{parentName:"li",href:"https://github.com/datahub-project/datahub/blob/master/metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py"},"GitHub"))),(0,n.yg)("h2",null,"Questions"),(0,n.yg)("p",null,"If you've got any questions on configuring ingestion for PowerBI, feel free to ping us on ",(0,n.yg)("a",{parentName:"p",href:"https://datahub.com/slack"},"our Slack"),"."))}f.isMDXComponent=!0}}]);
Reference in New Issue View Git Blame Copy Permalink