yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-14 02:11:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(metabase): use non deprecated fields in ChartInfo and DashboardInfo aspects (#15102)

Browse Source
This commit is contained in:
Sergio Gómez Villamor 2025-10-27 10:17:08 +01:00 committed by GitHub
parent ee20dbbcb3
commit f844ea7b2d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 147 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
metadata-ingestion/src/datahub/ingestion/source/metabase.py
View File

@ -52,6 +52,7 @@ from datahub.metadata.schema_classes import (
ChartQueryTypeClass, ChartQueryTypeClass,
ChartTypeClass, ChartTypeClass,
DashboardInfoClass, DashboardInfoClass,
EdgeClass,
OwnerClass, OwnerClass,
OwnershipClass, OwnershipClass,
OwnershipTypeClass, OwnershipTypeClass,
@ -338,19 +339,25 @@ class MetabaseSource(StatefulIngestionSourceBase):
lastModified=AuditStamp(time=modified_ts, actor=modified_actor), lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
) )
chart_urns = [] # Convert chart URNs to chart edges (instead of deprecated charts field)
chart_edges = []
cards_data = dashboard_details.get("dashcards", {}) cards_data = dashboard_details.get("dashcards", {})
for card_info in cards_data: for card_info in cards_data:
card_id = card_info.get("card").get("id", "") card_id = card_info.get("card").get("id", "")
if not card_id: if not card_id:
continue # most likely a virtual card without an id (text or heading), not relevant. continue # most likely a virtual card without an id (text or heading), not relevant.
chart_urn = builder.make_chart_urn(self.platform, str(card_id)) chart_urn = builder.make_chart_urn(self.platform, str(card_id))
chart_urns.append(chart_urn) chart_edges.append(
EdgeClass(
destinationUrn=chart_urn,
lastModified=last_modified.lastModified,
)
)
dashboard_info_class = DashboardInfoClass( dashboard_info_class = DashboardInfoClass(
description=description, description=description,
title=title, title=title,
charts=chart_urns, chartEdges=chart_edges,
lastModified=last_modified, lastModified=last_modified,
dashboardUrl=f"{self.config.display_uri}/dashboard/{dashboard_id}", dashboardUrl=f"{self.config.display_uri}/dashboard/{dashboard_id}",
customProperties={}, customProperties={},
@ -488,13 +495,25 @@ class MetabaseSource(StatefulIngestionSourceBase):
datasource_urn = self.get_datasource_urn(card_details) datasource_urn = self.get_datasource_urn(card_details)
custom_properties = self.construct_card_custom_properties(card_details) custom_properties = self.construct_card_custom_properties(card_details)
input_edges = (
[
EdgeClass(
destinationUrn=urn,
lastModified=last_modified.lastModified,
)
for urn in datasource_urn
]
if datasource_urn
else None
)
chart_info = ChartInfoClass( chart_info = ChartInfoClass(
type=chart_type, type=chart_type,
description=description, description=description,
title=title, title=title,
lastModified=last_modified, lastModified=last_modified,
chartUrl=f"{self.config.display_uri}/card/{card_id}", chartUrl=f"{self.config.display_uri}/card/{card_id}",
inputs=datasource_urn, inputEdges=input_edges,
customProperties=custom_properties, customProperties=custom_properties,
) )
chart_snapshot.aspects.append(chart_info) chart_snapshot.aspects.append(chart_info)

148
metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json
View File

@ -24,12 +24,20 @@
} }
}, },
"chartUrl": "http://localhost:3000/card/1", "chartUrl": "http://localhost:3000/card/1",
"inputs": [ "inputEdges": [
{ {
"string": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.customer,PROD)" "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.customer,PROD)",
"lastModified": {
"time": 1639417592792,
"actor": "urn:li:corpuser:admin@metabase.com"
}
}, },
{ {
"string": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.payment,PROD)" "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.payment,PROD)",
"lastModified": {
"time": 1639417592792,
"actor": "urn:li:corpuser:admin@metabase.com"
}
} }
], ],
"type": "TABLE" "type": "TABLE"
@ -91,9 +99,13 @@
} }
}, },
"chartUrl": "http://localhost:3000/card/2", "chartUrl": "http://localhost:3000/card/2",
"inputs": [ "inputEdges": [
{ {
"string": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.film,PROD)" "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,dvdrental.public.film,PROD)",
"lastModified": {
"time": 1636614000000,
"actor": "urn:li:corpuser:admin@metabase.com"
}
} }
], ],
"type": "BAR" "type": "BAR"
@ -149,12 +161,20 @@
} }
}, },
"chartUrl": "http://localhost:3000/card/3", "chartUrl": "http://localhost:3000/card/3",
"inputs": [ "inputEdges": [
{ {
"string": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.customer,PROD)" "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.customer,PROD)",
"lastModified": {
"time": 1685628119636,
"actor": "urn:li:corpuser:john.doe@example.com"
}
}, },
{ {
"string": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.payment,PROD)" "destinationUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,acryl-data.public.payment,PROD)",
"lastModified": {
"time": 1685628119636,
"actor": "urn:li:corpuser:john.doe@example.com"
}
} }
], ],
"type": "TABLE" "type": "TABLE"
@ -195,12 +215,32 @@
"customProperties": {}, "customProperties": {},
"title": "Dashboard 1", "title": "Dashboard 1",
"description": "", "description": "",
"charts": [ "charts": [],
"urn:li:chart:(metabase,1)", "chartEdges": [
"urn:li:chart:(metabase,2)", {
"urn:li:chart:(metabase,3)" "destinationUrn": "urn:li:chart:(metabase,1)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,2)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,3)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
}
], ],
"datasets": [], "datasets": [],
"dashboards": [],
"lastModified": { "lastModified": {
"created": { "created": {
"time": 1705398694904, "time": 1705398694904,
@ -249,12 +289,32 @@
"customProperties": {}, "customProperties": {},
"title": "Dashboard 2", "title": "Dashboard 2",
"description": "", "description": "",
"charts": [ "charts": [],
"urn:li:chart:(metabase,1)", "chartEdges": [
"urn:li:chart:(metabase,2)", {
"urn:li:chart:(metabase,3)" "destinationUrn": "urn:li:chart:(metabase,1)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,2)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,3)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
}
], ],
"datasets": [], "datasets": [],
"dashboards": [],
"lastModified": { "lastModified": {
"created": { "created": {
"time": 1705398694904, "time": 1705398694904,
@ -303,12 +363,32 @@
"customProperties": {}, "customProperties": {},
"title": "Dashboard 1", "title": "Dashboard 1",
"description": "", "description": "",
"charts": [ "charts": [],
"urn:li:chart:(metabase,1)", "chartEdges": [
"urn:li:chart:(metabase,2)", {
"urn:li:chart:(metabase,3)" "destinationUrn": "urn:li:chart:(metabase,1)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,2)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,3)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
}
], ],
"datasets": [], "datasets": [],
"dashboards": [],
"lastModified": { "lastModified": {
"created": { "created": {
"time": 1705398694904, "time": 1705398694904,
@ -357,12 +437,32 @@
"customProperties": {}, "customProperties": {},
"title": "Dashboard 2", "title": "Dashboard 2",
"description": "", "description": "",
"charts": [ "charts": [],
"urn:li:chart:(metabase,1)", "chartEdges": [
"urn:li:chart:(metabase,2)", {
"urn:li:chart:(metabase,3)" "destinationUrn": "urn:li:chart:(metabase,1)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,2)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
},
{
"destinationUrn": "urn:li:chart:(metabase,3)",
"lastModified": {
"time": 1705398694904,
"actor": "urn:li:corpuser:admin@metabase.com"
}
}
], ],
"datasets": [], "datasets": [],
"dashboards": [],
"lastModified": { "lastModified": {
"created": { "created": {
"time": 1705398694904, "time": 1705398694904,