yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-13 11:06:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ui): Support PowerBI Ingestion via UI form (#7817)

Browse Source
This commit is contained in:
John Joyce 2023-04-17 11:37:00 -07:00 committed by GitHub
parent 9c726b3a56
commit ed85296e4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 208 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts
View File

@ -83,7 +83,7 @@ import {
PROJECT_NAME,
} from './lookml';
import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto';
import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, UNITY_CATALOG } from '../constants';
import { BIGQUERY_BETA, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG } from '../constants';
import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql';
import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql';
@ -116,6 +116,19 @@ import {
TARGET_PLATFORM_INSTANCE,
DBT_CLOUD_TOKEN,
} from './dbt_cloud';
import {
ADMIN_APIS_ONLY,
EXTRACT_ENDORSEMENTS_AS_TAGS,
EXTRACT_OWNERSHIP,
INCLUDE_REPORTS,
INCLUDE_POWERBI_LINEAGE,
INCLUDE_WORKSPACES,
POWERBI_CLIENT_ID,
POWERBI_CLIENT_SECRET,
POWERBI_TENANT_ID,
WORKSPACE_ID_ALLOW,
WORKSPACE_ID_DENY,
} from './powerbi';
export enum RecipeSections {
Connection = 0,
@ -401,6 +414,20 @@ export const RECIPE_FIELDS: RecipeFields = {
],
filterSectionTooltip: 'Include or exclude specific dbt Node (resources) from ingestion.',
},
[POWER_BI]: {
fields: [POWERBI_TENANT_ID, POWERBI_CLIENT_ID, POWERBI_CLIENT_SECRET],
filterFields: [WORKSPACE_ID_ALLOW, WORKSPACE_ID_DENY],
advancedFields: [
INCLUDE_WORKSPACES,
INCLUDE_REPORTS,
INCLUDE_POWERBI_LINEAGE,
EXTRACT_OWNERSHIP,
EXTRACT_ENDORSEMENTS_AS_TAGS,
ADMIN_APIS_ONLY,
STATEFUL_INGESTION_ENABLED,
],
filterSectionTooltip: 'Include or exclude specific PowerBI Workspaces from ingestion.',
},
};
export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));

172
datahub-web-react/src/app/ingest/source/builder/RecipeForm/powerbi.ts Normal file
View File

@ -0,0 +1,172 @@
import { get } from 'lodash';
import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
export const POWERBI_CLIENT_ID: RecipeField = {
name: 'client_id',
label: 'Client ID',
tooltip: 'The Azure AD Client ID',
type: FieldType.TEXT,
fieldPath: 'source.config.client_id',
placeholder: 'client id',
required: true,
rules: null,
};
export const POWERBI_CLIENT_SECRET: RecipeField = {
name: 'client_secret',
label: 'Client Secret',
tooltip: 'The Azure AD Client Secret',
type: FieldType.SECRET,
fieldPath: 'source.config.client_secret',
placeholder: 'client secret',
required: true,
rules: null,
};
export const POWERBI_TENANT_ID: RecipeField = {
name: 'tenant_id',
label: 'Tenant ID',
tooltip: 'The Azure tenant id for your PowerBI instance.',
type: FieldType.TEXT,
fieldPath: 'source.config.tenant_id',
placeholder: 'a949d688-67c0-4bf1-a344-e939411c6c0a',
required: true,
rules: null,
};
const includeWorkspacesPath = 'source.config.extract_workspaces_to_containers';
export const INCLUDE_WORKSPACES: RecipeField = {
name: 'extract_workspaces_to_containers',
label: 'Include Workspaces',
tooltip: 'Extract PowerBI Workspaces as Container nodes in DataHub.',
type: FieldType.BOOLEAN,
fieldPath: includeWorkspacesPath,
getValueFromRecipeOverride: (recipe: any) => {
const includeWorkspaces = get(recipe, includeWorkspacesPath);
if (includeWorkspaces !== undefined && includeWorkspaces !== null) {
return includeWorkspaces;
}
return true;
},
rules: null,
};
const adminsApisOnlyPath = 'source.config.admin_apis_only';
export const ADMIN_APIS_ONLY: RecipeField = {
name: 'admin_apis_only',
label: 'Use Admin APIs Only',
tooltip:
'Only use the Admin APIs for extracting information from PowerBI. See the DataHub Ingestion Source Guide for more information.',
type: FieldType.BOOLEAN,
fieldPath: adminsApisOnlyPath,
getValueFromRecipeOverride: (recipe: any) => {
const adminApisOnly = get(recipe, adminsApisOnlyPath);
if (adminApisOnly !== undefined && adminApisOnly !== null) {
return adminApisOnly;
}
return false;
},
rules: null,
};
const includeReportsPath = 'source.config.extract_reports';
export const INCLUDE_REPORTS: RecipeField = {
name: 'extract_reports',
label: 'Include Reports',
tooltip: 'Extract PowerBI Reports as Dashboard nodes in DataHub.',
type: FieldType.BOOLEAN,
fieldPath: includeReportsPath,
getValueFromRecipeOverride: (recipe: any) => {
const includeReports = get(recipe, includeReportsPath);
if (includeReports !== undefined && includeReports !== null) {
return includeReports;
}
return true;
},
rules: null,
};
const includeLineagePath = 'source.config.extract_lineage';
export const INCLUDE_POWERBI_LINEAGE: RecipeField = {
name: 'include_powerbi_lineage',
label: 'Include Lineage',
tooltip: 'Extract lineage between PowerBI Datasets and the External Tables from which they are derived.',
type: FieldType.BOOLEAN,
fieldPath: includeLineagePath,
getValueFromRecipeOverride: (recipe: any) => {
const includeLineage = get(recipe, includeLineagePath);
if (includeLineage !== undefined && includeLineage !== null) {
return includeLineage;
}
return true;
},
rules: null,
};
const extractEndorsementsAsTags = 'source.config.extract_endorsements_to_tags';
export const EXTRACT_ENDORSEMENTS_AS_TAGS: RecipeField = {
name: 'extract_endorsements_to_tags',
label: 'Extract Endorsements as Tags',
tooltip: 'Extract PowerBI Endorsements as DataHub Tags.',
type: FieldType.BOOLEAN,
fieldPath: extractEndorsementsAsTags,
getValueFromRecipeOverride: (recipe: any) => {
const extractEndorsements = get(recipe, extractEndorsementsAsTags);
if (extractEndorsements !== undefined && extractEndorsements !== null) {
return extractEndorsements;
}
return false;
},
rules: null,
};
const extractOwnershipPath = 'source.config.extract_ownership';
export const EXTRACT_OWNERSHIP: RecipeField = {
name: 'extract_ownership',
label: 'Extract Ownership',
tooltip:
'Extract Owner Metadata defined inside PowerBI. Admin API access is required if this setting is enabled. Note that enabling this may overwrite owners that you add inside DataHub.',
type: FieldType.BOOLEAN,
fieldPath: extractOwnershipPath,
getValueFromRecipeOverride: (recipe: any) => {
const extractOwnership = get(recipe, extractOwnershipPath);
if (extractOwnership !== undefined && extractOwnership !== null) {
return extractOwnership;
}
return false;
},
rules: null,
};
const workspaceIdAllowFieldPath = 'source.config.workspace_id_pattern.allow';
export const WORKSPACE_ID_ALLOW: RecipeField = {
name: 'workspace_id_pattern.allow',
label: 'Allow Patterns',
tooltip:
'Only include specific Workspaces by providing the unique id of a Workspace, or a Regular Expression (REGEX). If not provided, all Workspaces will be included.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: workspaceIdAllowFieldPath,
rules: null,
section: 'Workspaces',
placeholder: '4bd10256-e999-45dd-8e56-571c77153a5f',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, workspaceIdAllowFieldPath),
};
const workspaceIdDenyFieldPath = 'source.config.workspace_id_pattern.deny';
export const WORKSPACE_ID_DENY: RecipeField = {
name: 'workspace_id_pattern.deny',
label: 'Deny Patterns',
tooltip:
'Exclude specific Workspaces by providing the unique id of a Workspace, or a Regular Expression (REGEX). If not provided, all Workspaces will be included. Deny patterns always take precendence over Allow patterns.',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: workspaceIdDenyFieldPath,
rules: null,
section: 'Workspaces',
placeholder: '4bd10256-e999-45dd-8e56-571c77153a5f',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, workspaceIdDenyFieldPath),
};

14
datahub-web-react/src/app/ingest/source/builder/sources.json
View File

@ -48,6 +48,13 @@
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/tableau/",
"recipe": "source:\n type: tableau\n config:\n # Coordinates\n connect_uri: null\n stateful_ingestion:\n enabled: true"
},
{
"urn": "urn:li:dataPlatform:powerbi",
"name": "powerbi",
"displayName": "PowerBI",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/",
"recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: null\n # Your Power BI client id\n client_id: null\n # Your Power BI client secret\n client_secret: null\n stateful_ingestion:\n enabled: true"
},
{
"urn": "urn:li:dataPlatform:dbt",
"name": "dbt-cloud",
@ -167,13 +174,6 @@
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/",
"recipe": "source:\n type: metabase\n config:\n # Coordinates\n connect_uri:\n\n # Credentials\n username: root\n password: example"
},
{
"urn": "urn:li:dataPlatform:powerbi",
"name": "powerbi",
"displayName": "Power BI",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/powerbi/",
"recipe": "source:\n type: \"powerbi\"\n config:\n # Your Power BI tenant identifier\n tenant_id: a949d688-67c0-4bf1-a344-e939411c6c0a\n # Ingest elements of below PowerBi Workspace into Datahub\n workspace_id: 4bd10256-e999-45dd-8e56-571c77153a5f\n # Workspaces dataset environments (PROD, DEV, QA, STAGE)\n env: DEV\n # Azure AD Application identifier\n client_id: foo\n # Azure AD App client secret\n client_secret: bar\n # Enable / Disable ingestion of user information for dashboards\n extract_ownership: true\n # dataset_type_mapping is fixed mapping of Power BI datasources type to equivalent Datahub \"data platform\" dataset\n dataset_type_mapping:\n PostgreSql: postgres\n Oracle: oracle"
},
{
"urn": "urn:li:dataPlatform:mode",
"name": "mode",