diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index e4609c0586..44f48fa072 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -83,7 +83,7 @@ import { PROJECT_NAME, } from './lookml'; import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto'; -import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, UNITY_CATALOG } from '../constants'; +import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG } from '../constants'; import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta'; import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql'; import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql'; @@ -116,6 +116,19 @@ import { TARGET_PLATFORM_INSTANCE, DBT_CLOUD_TOKEN, } from './dbt_cloud'; +import { + ADMIN_APIS_ONLY, + EXTRACT_ENDORSEMENTS_AS_TAGS, + EXTRACT_OWNERSHIP, + INCLUDE_REPORTS, + INCLUDE_POWERBI_LINEAGE, + INCLUDE_WORKSPACES, + POWERBI_CLIENT_ID, + POWERBI_CLIENT_SECRET, + POWERBI_TENANT_ID, + WORKSPACE_ID_ALLOW, + WORKSPACE_ID_DENY, +} from './powerbi'; export enum RecipeSections { Connection = 0, @@ -401,6 +414,20 @@ export const RECIPE_FIELDS: RecipeFields = { ], filterSectionTooltip: 'Include or exclude specific dbt Node (resources) from ingestion.', }, + [POWER_BI]: { + fields: [POWERBI_TENANT_ID, POWERBI_CLIENT_ID, POWERBI_CLIENT_SECRET], + filterFields: [WORKSPACE_ID_ALLOW, WORKSPACE_ID_DENY], + advancedFields: [ + INCLUDE_WORKSPACES, + INCLUDE_REPORTS, + INCLUDE_POWERBI_LINEAGE, + EXTRACT_OWNERSHIP, + EXTRACT_ENDORSEMENTS_AS_TAGS, + ADMIN_APIS_ONLY, + STATEFUL_INGESTION_ENABLED, + ], + filterSectionTooltip: 'Include or exclude specific PowerBI Workspaces from ingestion.', + }, }; export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS)); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/powerbi.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/powerbi.ts new file mode 100644 index 0000000000..ecd0783600 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/powerbi.ts @@ -0,0 +1,172 @@ +import { get } from 'lodash'; + +import { RecipeField, FieldType, setListValuesOnRecipe } from './common'; + +export const POWERBI_CLIENT_ID: RecipeField = { + name: 'client_id', + label: 'Client ID', + tooltip: 'The Azure AD Client ID', + type: FieldType.TEXT, + fieldPath: 'source.config.client_id', + placeholder: 'client id', + required: true, + rules: null, +}; + +export const POWERBI_CLIENT_SECRET: RecipeField = { + name: 'client_secret', + label: 'Client Secret', + tooltip: 'The Azure AD Client Secret', + type: FieldType.SECRET, + fieldPath: 'source.config.client_secret', + placeholder: 'client secret', + required: true, + rules: null, +}; + +export const POWERBI_TENANT_ID: RecipeField = { + name: 'tenant_id', + label: 'Tenant ID', + tooltip: 'The Azure tenant id for your PowerBI instance.', + type: FieldType.TEXT, + fieldPath: 'source.config.tenant_id', + placeholder: 'a949d688-67c0-4bf1-a344-e939411c6c0a', + required: true, + rules: null, +}; + +const includeWorkspacesPath = 'source.config.extract_workspaces_to_containers'; +export const INCLUDE_WORKSPACES: RecipeField = { + name: 'extract_workspaces_to_containers', + label: 'Include Workspaces', + tooltip: 'Extract PowerBI Workspaces as Container nodes in DataHub.', + type: FieldType.BOOLEAN, + fieldPath: includeWorkspacesPath, + getValueFromRecipeOverride: (recipe: any) => { + const includeWorkspaces = get(recipe, includeWorkspacesPath); + if (includeWorkspaces !== undefined && includeWorkspaces !== null) { + return includeWorkspaces; + } + return true; + }, + rules: null, +}; + +const adminsApisOnlyPath = 'source.config.admin_apis_only'; +export const ADMIN_APIS_ONLY: RecipeField = { + name: 'admin_apis_only', + label: 'Use Admin APIs Only', + tooltip: + 'Only use the Admin APIs for extracting information from PowerBI. See the DataHub Ingestion Source Guide for more information.', + type: FieldType.BOOLEAN, + fieldPath: adminsApisOnlyPath, + getValueFromRecipeOverride: (recipe: any) => { + const adminApisOnly = get(recipe, adminsApisOnlyPath); + if (adminApisOnly !== undefined && adminApisOnly !== null) { + return adminApisOnly; + } + return false; + }, + rules: null, +}; + +const includeReportsPath = 'source.config.extract_reports'; +export const INCLUDE_REPORTS: RecipeField = { + name: 'extract_reports', + label: 'Include Reports', + tooltip: 'Extract PowerBI Reports as Dashboard nodes in DataHub.', + type: FieldType.BOOLEAN, + fieldPath: includeReportsPath, + getValueFromRecipeOverride: (recipe: any) => { + const includeReports = get(recipe, includeReportsPath); + if (includeReports !== undefined && includeReports !== null) { + return includeReports; + } + return true; + }, + rules: null, +}; + +const includeLineagePath = 'source.config.extract_lineage'; +export const INCLUDE_POWERBI_LINEAGE: RecipeField = { + name: 'include_powerbi_lineage', + label: 'Include Lineage', + tooltip: 'Extract lineage between PowerBI Datasets and the External Tables from which they are derived.', + type: FieldType.BOOLEAN, + fieldPath: includeLineagePath, + getValueFromRecipeOverride: (recipe: any) => { + const includeLineage = get(recipe, includeLineagePath); + if (includeLineage !== undefined && includeLineage !== null) { + return includeLineage; + } + return true; + }, + rules: null, +}; + +const extractEndorsementsAsTags = 'source.config.extract_endorsements_to_tags'; +export const EXTRACT_ENDORSEMENTS_AS_TAGS: RecipeField = { + name: 'extract_endorsements_to_tags', + label: 'Extract Endorsements as Tags', + tooltip: 'Extract PowerBI Endorsements as DataHub Tags.', + type: FieldType.BOOLEAN, + fieldPath: extractEndorsementsAsTags, + getValueFromRecipeOverride: (recipe: any) => { + const extractEndorsements = get(recipe, extractEndorsementsAsTags); + if (extractEndorsements !== undefined && extractEndorsements !== null) { + return extractEndorsements; + } + return false; + }, + rules: null, +}; + +const extractOwnershipPath = 'source.config.extract_ownership'; +export const EXTRACT_OWNERSHIP: RecipeField = { + name: 'extract_ownership', + label: 'Extract Ownership', + tooltip: + 'Extract Owner Metadata defined inside PowerBI. Admin API access is required if this setting is enabled. Note that enabling this may overwrite owners that you add inside DataHub.', + type: FieldType.BOOLEAN, + fieldPath: extractOwnershipPath, + getValueFromRecipeOverride: (recipe: any) => { + const extractOwnership = get(recipe, extractOwnershipPath); + if (extractOwnership !== undefined && extractOwnership !== null) { + return extractOwnership; + } + return false; + }, + rules: null, +}; + +const workspaceIdAllowFieldPath = 'source.config.workspace_id_pattern.allow'; +export const WORKSPACE_ID_ALLOW: RecipeField = { + name: 'workspace_id_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Workspaces by providing the unique id of a Workspace, or a Regular Expression (REGEX). If not provided, all Workspaces will be included.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: workspaceIdAllowFieldPath, + rules: null, + section: 'Workspaces', + placeholder: '4bd10256-e999-45dd-8e56-571c77153a5f', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, workspaceIdAllowFieldPath), +}; + +const workspaceIdDenyFieldPath = 'source.config.workspace_id_pattern.deny'; +export const WORKSPACE_ID_DENY: RecipeField = { + name: 'workspace_id_pattern.deny', + label: 'Deny Patterns', + tooltip: + 'Exclude specific Workspaces by providing the unique id of a Workspace, or a Regular Expression (REGEX). If not provided, all Workspaces will be included. Deny patterns always take precendence over Allow patterns.', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: workspaceIdDenyFieldPath, + rules: null, + section: 'Workspaces', + placeholder: '4bd10256-e999-45dd-8e56-571c77153a5f', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, workspaceIdDenyFieldPath), +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 38aaeba165..c9db0433b3 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -48,6 +48,13 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/tableau/", "recipe": "source:\n type: tableau\n config:\n # Coordinates\n connect_uri: null\n stateful_ingestion:\n enabled: true" }, + { + "urn": "urn:li:dataPlatform:powerbi", + "name": "powerbi", + "displayName": "PowerBI", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/", + "recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: null\n # Your Power BI client id\n client_id: null\n # Your Power BI client secret\n client_secret: null\n stateful_ingestion:\n enabled: true" + }, { "urn": "urn:li:dataPlatform:dbt", "name": "dbt-cloud", @@ -167,13 +174,6 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/", "recipe": "source:\n type: metabase\n config:\n # Coordinates\n connect_uri:\n\n # Credentials\n username: root\n password: example" }, - { - "urn": "urn:li:dataPlatform:powerbi", - "name": "powerbi", - "displayName": "Power BI", - "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/", - "recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: a949d688-67c0-4bf1-a344-e939411c6c0a\n # Ingest elements of below PowerBi Workspace into Datahub\n workspace_id: 4bd10256-e999-45dd-8e56-571c77153a5f\n # Workspaces dataset environments (PROD, DEV, QA, STAGE)\n env: DEV\n # Azure AD Application identifier\n client_id: foo\n # Azure AD App client secret\n client_secret: bar\n # Enable / Disable ingestion of user information for dashboards\n extract_ownership: true\n # dataset_type_mapping is fixed mapping of Power BI datasources type to equivalent Datahub \"data platform\" dataset\n dataset_type_mapping:\n PostgreSql: postgres\n Oracle: oracle" - }, { "urn": "urn:li:dataPlatform:mode", "name": "mode", @@ -209,4 +209,4 @@ "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/", "recipe": "source:\n type: \n config:\n # Source-type specifics config\n " } -] +] \ No newline at end of file