yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-08-24 17:18:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): powerbi # store powerbi entity descriptions (#7154)

Browse Source 
Co-authored-by: John Joyce <john@acryl.io>
This commit is contained in:
Teppo Naakka 2023-03-13 22:31:21 +02:00 committed by GitHub
parent fbfe43b1cb
commit 23830423f8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 114 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
metadata-ingestion/src/datahub/ingestion/source/powerbi/powerbi.py
View File

@ -242,7 +242,8 @@ class Mapper:
logger.debug(f"{Constant.Dataset_URN}={ds_urn}") logger.debug(f"{Constant.Dataset_URN}={ds_urn}")
# Create datasetProperties mcp # Create datasetProperties mcp
ds_properties = DatasetPropertiesClass( ds_properties = DatasetPropertiesClass(
name=table.name, description=table.name name=table.name,
description=dataset.description,
) )
info_mcp = self.new_mcp( info_mcp = self.new_mcp(
@ -415,7 +416,7 @@ class Mapper:
# DashboardInfo mcp # DashboardInfo mcp
dashboard_info_cls = DashboardInfoClass( dashboard_info_cls = DashboardInfoClass(
description=dashboard.displayName or "", description=dashboard.description,
title=dashboard.displayName or "", title=dashboard.displayName or "",
charts=chart_urn_list, charts=chart_urn_list,
lastModified=ChangeAuditStamps(), lastModified=ChangeAuditStamps(),
@ -748,7 +749,7 @@ class Mapper:
# DashboardInfo mcp # DashboardInfo mcp
dashboard_info_cls = DashboardInfoClass( dashboard_info_cls = DashboardInfoClass(
description=report.description or "", description=report.description,
title=report.name or "", title=report.name or "",
charts=chart_urn_list, charts=chart_urn_list,
lastModified=ChangeAuditStamps(), lastModified=ChangeAuditStamps(),

3
metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_classes.py
View File

@ -63,6 +63,7 @@ class Table:
class PowerBIDataset: class PowerBIDataset:
id: str id: str
name: Optional[str] name: Optional[str]
description: str
webUrl: Optional[str] webUrl: Optional[str]
workspace_id: str workspace_id: str
parameters: Optional[Dict[str, str]] parameters: Optional[Dict[str, str]]
@ -159,6 +160,7 @@ class Tile:
class Dashboard: class Dashboard:
id: str id: str
displayName: str displayName: str
description: str
embedUrl: str embedUrl: str
webUrl: Optional[str] webUrl: Optional[str]
isReadOnly: Any isReadOnly: Any
@ -187,6 +189,7 @@ def new_powerbi_dataset(workspace_id: str, raw_instance: dict) -> PowerBIDataset
return PowerBIDataset( return PowerBIDataset(
id=raw_instance["id"], id=raw_instance["id"],
name=raw_instance.get("name"), name=raw_instance.get("name"),
description=raw_instance.get("description", str()),
webUrl="{}/details".format(raw_instance.get("webUrl")) webUrl="{}/details".format(raw_instance.get("webUrl"))
if raw_instance.get("webUrl") is not None if raw_instance.get("webUrl") is not None
else None, else None,

3
metadata-ingestion/src/datahub/ingestion/source/powerbi/rest_api_wrapper/data_resolver.py
View File

@ -167,6 +167,7 @@ class DataResolverBase(ABC):
id=instance.get(Constant.ID), id=instance.get(Constant.ID),
isReadOnly=instance.get(Constant.IS_READ_ONLY), isReadOnly=instance.get(Constant.IS_READ_ONLY),
displayName=instance.get(Constant.DISPLAY_NAME), displayName=instance.get(Constant.DISPLAY_NAME),
description=instance.get(Constant.DESCRIPTION, str()),
embedUrl=instance.get(Constant.EMBED_URL), embedUrl=instance.get(Constant.EMBED_URL),
webUrl=instance.get(Constant.WEB_URL), webUrl=instance.get(Constant.WEB_URL),
workspace_id=workspace.id, workspace_id=workspace.id,
@ -249,7 +250,7 @@ class DataResolverBase(ABC):
name=raw_instance.get(Constant.NAME), name=raw_instance.get(Constant.NAME),
webUrl=raw_instance.get(Constant.WEB_URL), webUrl=raw_instance.get(Constant.WEB_URL),
embedUrl=raw_instance.get(Constant.EMBED_URL), embedUrl=raw_instance.get(Constant.EMBED_URL),
description=raw_instance.get(Constant.DESCRIPTION), description=raw_instance.get(Constant.DESCRIPTION, str()),
pages=self._get_pages_by_report( pages=self._get_pages_by_report(
workspace=workspace, report_id=raw_instance[Constant.ID] workspace=workspace, report_id=raw_instance[Constant.ID]
), ),

2
metadata-ingestion/tests/integration/powerbi/golden_test_admin_access_not_allowed.json
View File

@ -217,7 +217,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"

30
metadata-ingestion/tests/integration/powerbi/golden_test_admin_only.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "",
"tags": [] "tags": []
} }
}, },
@ -98,7 +98,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "",
"tags": [] "tags": []
} }
}, },
@ -155,7 +155,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "",
"tags": [] "tags": []
} }
}, },
@ -252,7 +252,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "",
"tags": [] "tags": []
} }
}, },
@ -309,7 +309,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "",
"tags": [] "tags": []
} }
}, },
@ -366,7 +366,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "",
"tags": [] "tags": []
} }
}, },
@ -423,7 +423,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "",
"tags": [] "tags": []
} }
}, },
@ -717,7 +717,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
@ -809,7 +809,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "",
"tags": [] "tags": []
} }
}, },
@ -842,7 +842,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "",
"tags": [] "tags": []
} }
}, },
@ -899,7 +899,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "",
"tags": [] "tags": []
} }
}, },
@ -956,7 +956,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "",
"tags": [] "tags": []
} }
}, },
@ -1053,7 +1053,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "",
"tags": [] "tags": []
} }
}, },
@ -1110,7 +1110,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "",
"tags": [] "tags": []
} }
}, },

30
metadata-ingestion/tests/integration/powerbi/golden_test_container.json
View File

@ -74,7 +74,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -122,7 +122,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -170,7 +170,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -218,7 +218,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -266,7 +266,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -314,7 +314,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -362,7 +362,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -410,7 +410,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -725,7 +725,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
@ -832,7 +832,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -880,7 +880,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -928,7 +928,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -976,7 +976,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -1024,7 +1024,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -1072,7 +1072,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },

18
metadata-ingestion/tests/integration/powerbi/golden_test_disabled_ownership.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -74,7 +74,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -107,7 +107,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -140,7 +140,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -173,7 +173,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -206,7 +206,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -239,7 +239,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -479,7 +479,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"

18
metadata-ingestion/tests/integration/powerbi/golden_test_endorsement.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -60,7 +60,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -112,7 +112,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -164,7 +164,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -216,7 +216,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -268,7 +268,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -320,7 +320,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -353,7 +353,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -623,7 +623,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"

18
metadata-ingestion/tests/integration/powerbi/golden_test_ingest.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -74,7 +74,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -107,7 +107,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -140,7 +140,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -173,7 +173,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -206,7 +206,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -239,7 +239,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -509,7 +509,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"

18
metadata-ingestion/tests/integration/powerbi/golden_test_lineage.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -98,7 +98,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -155,7 +155,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -252,7 +252,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -309,7 +309,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -366,7 +366,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -423,7 +423,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -717,7 +717,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"

18
metadata-ingestion/tests/integration/powerbi/golden_test_lower_case_urn_ingest.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -74,7 +74,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -107,7 +107,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -140,7 +140,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -173,7 +173,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -206,7 +206,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -239,7 +239,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -509,7 +509,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"

30
metadata-ingestion/tests/integration/powerbi/golden_test_report.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -74,7 +74,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -107,7 +107,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -140,7 +140,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -173,7 +173,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -206,7 +206,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -239,7 +239,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -509,7 +509,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
@ -601,7 +601,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -634,7 +634,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -667,7 +667,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -700,7 +700,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -733,7 +733,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -766,7 +766,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },

20
metadata-ingestion/tests/integration/powerbi/golden_test_scan_all_workspaces.json
View File

@ -8,7 +8,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "public issue_history", "name": "public issue_history",
"description": "public issue_history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -41,7 +41,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "SNOWFLAKE_TESTTABLE", "name": "SNOWFLAKE_TESTTABLE",
"description": "SNOWFLAKE_TESTTABLE", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -74,7 +74,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query", "name": "snowflake native-query",
"description": "snowflake native-query", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -107,7 +107,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "snowflake native-query-with-join", "name": "snowflake native-query-with-join",
"description": "snowflake native-query-with-join", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -140,7 +140,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "job-history", "name": "job-history",
"description": "job-history", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -173,7 +173,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "postgres_test_table", "name": "postgres_test_table",
"description": "postgres_test_table", "description": "Library dataset description",
"tags": [] "tags": []
} }
}, },
@ -206,7 +206,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "dbo_book_issue", "name": "dbo_book_issue",
"description": "dbo_book_issue", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -239,7 +239,7 @@
"json": { "json": {
"customProperties": {}, "customProperties": {},
"name": "ms_sql_native_table", "name": "ms_sql_native_table",
"description": "ms_sql_native_table", "description": "hr pbi test description",
"tags": [] "tags": []
} }
}, },
@ -479,7 +479,7 @@
"workspaceId": "64ED5CAD-7C10-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C10-4684-8180-826122881108"
}, },
"title": "test_dashboard", "title": "test_dashboard",
"description": "test_dashboard", "description": "Description of test dashboard",
"charts": [ "charts": [
"urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)", "urn:li:chart:(powerbi,charts.B8E293DC-0C83-4AA0-9BB9-0A8738DF24A0)",
"urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)" "urn:li:chart:(powerbi,charts.23212598-23b5-4980-87cc-5fc0ecd84385)"
@ -564,7 +564,7 @@
"workspaceId": "64ED5CAD-7C22-4684-8180-826122881108" "workspaceId": "64ED5CAD-7C22-4684-8180-826122881108"
}, },
"title": "test_dashboard2", "title": "test_dashboard2",
"description": "test_dashboard2", "description": "",
"charts": [], "charts": [],
"datasets": [], "datasets": [],
"lastModified": { "lastModified": {

4
metadata-ingestion/tests/integration/powerbi/test_powerbi.py
View File

@ -82,6 +82,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE", "id": "7D668CAD-7FFC-4505-9215-655BCA5BEBAE",
"isReadOnly": True, "isReadOnly": True,
"displayName": "test_dashboard", "displayName": "test_dashboard",
"description": "Description of test dashboard",
"embedUrl": "https://localhost/dashboards/embed/1", "embedUrl": "https://localhost/dashboards/embed/1",
"webUrl": "https://localhost/dashboards/web/1", "webUrl": "https://localhost/dashboards/web/1",
} }
@ -206,6 +207,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"json": { "json": {
"id": "05169CD2-E713-41E6-9600-1D8066D95445", "id": "05169CD2-E713-41E6-9600-1D8066D95445",
"name": "library-dataset", "name": "library-dataset",
"description": "Library dataset description",
"webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445", "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/05169CD2-E713-41E6-9600-1D8066D95445",
}, },
}, },
@ -215,6 +217,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"json": { "json": {
"id": "05169CD2-E713-41E6-96AA-1D8066D95445", "id": "05169CD2-E713-41E6-96AA-1D8066D95445",
"name": "library-dataset", "name": "library-dataset",
"description": "Library dataset description",
"webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445", "webUrl": "http://localhost/groups/64ED5CAD-7C22-4684-8180-826122881108/datasets/05169CD2-E713-41E6-96AA-1D8066D95445",
}, },
}, },
@ -224,6 +227,7 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"json": { "json": {
"id": "ba0130a1-5b03-40de-9535-b34e778ea6ed", "id": "ba0130a1-5b03-40de-9535-b34e778ea6ed",
"name": "hr_pbi_test", "name": "hr_pbi_test",
"description": "hr pbi test description",
"webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed", "webUrl": "http://localhost/groups/64ED5CAD-7C10-4684-8180-826122881108/datasets/ba0130a1-5b03-40de-9535-b34e778ea6ed",
}, },
}, },