yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-04 14:44:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(superset): add some custom properties for dashboards (#9670)

Browse Source
This commit is contained in:
Quentin FLEURENT NAMBOT 2024-01-20 10:24:10 +01:00 committed by GitHub
parent d78db0abee
commit 20ad83d0b1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 78 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
metadata-ingestion/src/datahub/ingestion/source/superset.py
View File

@ -267,13 +267,35 @@ class SupersetSource(StatefulIngestionSourceBase):
f"urn:li:chart:({self.platform},{value.get('meta', {}).get('chartId', 'unknown')})"
)
# Build properties
custom_properties = {
"Status": str(dashboard_data.get("status")),
"IsPublished": str(dashboard_data.get("published", False)).lower(),
"Owners": ", ".join(
map(
lambda owner: owner.get("username", "unknown"),
dashboard_data.get("owners", []),
)
),
"IsCertified": str(
True if dashboard_data.get("certified_by") else False
).lower(),
}
if dashboard_data.get("certified_by"):
custom_properties["CertifiedBy"] = dashboard_data.get("certified_by")
custom_properties["CertificationDetails"] = str(
dashboard_data.get("certification_details")
)
# Create DashboardInfo object
dashboard_info = DashboardInfoClass(
description="",
title=title,
charts=chart_urns,
lastModified=last_modified,
dashboardUrl=dashboard_url,
customProperties={},
customProperties=custom_properties,
)
dashboard_snapshot.aspects.append(dashboard_info)
return dashboard_snapshot

16
metadata-ingestion/tests/integration/superset/golden_test_ingest.json
View File

@ -11,7 +11,14 @@
},
{
"com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
"customProperties": {},
"customProperties": {
"Status": "published",
"IsPublished": "true",
"Owners": "test_username_1, test_username_2",
"IsCertified": "true",
"CertifiedBy": "Certification team",
"CertificationDetails": "Approved"
},
"title": "test_dashboard_title_1",
"description": "",
"charts": [
@ -52,7 +59,12 @@
},
{
"com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
"customProperties": {},
"customProperties": {
"Status": "draft",
"IsPublished": "false",
"Owners": "unknown",
"IsCertified": "false"
},
"title": "test_dashboard_title_2",
"description": "",
"charts": [

9
metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json
View File

@ -11,7 +11,14 @@
},
{
"com.linkedin.pegasus2avro.dashboard.DashboardInfo": {
"customProperties": {},
"customProperties": {
"Status": "published",
"IsPublished": "true",
"Owners": "test_username_1, test_username_2",
"IsCertified": "true",
"CertifiedBy": "Certification team",
"CertificationDetails": "Approved"
},
"title": "test_dashboard_title_1",
"description": "",
"charts": [

35
metadata-ingestion/tests/integration/superset/test_superset.py
View File

@ -41,6 +41,18 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"dashboard_title": "test_dashboard_title_1",
"url": "/dashboard/test_dashboard_url_1",
"position_json": '{"CHART-test-1": {"meta": { "chartId": "10" }}, "CHART-test-2": {"meta": { "chartId": "11" }}}',
"status": "published",
"published": True,
"owners": [
{
"username": "test_username_1",
},
{
"username": "test_username_2",
},
],
"certified_by": "Certification team",
"certification_details": "Approved",
},
{
"id": "2",
@ -51,6 +63,15 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
"dashboard_title": "test_dashboard_title_2",
"url": "/dashboard/test_dashboard_url_2",
"position_json": '{"CHART-test-3": {"meta": { "chartId": "12" }}, "CHART-test-4": {"meta": { "chartId": "13" }}}',
"status": "draft",
"published": False,
"owners": [
{
"first_name": "name",
},
],
"certified_by": "",
"certification_details": "",
},
],
},
@ -151,7 +172,6 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None:
@freeze_time(FROZEN_TIME)
@pytest.mark.integration
def test_superset_ingest(pytestconfig, tmp_path, mock_time, requests_mock):
test_resources_dir = pytestconfig.rootpath / "tests/integration/superset"
register_mock_api(request_mock=requests_mock)
@ -193,7 +213,6 @@ def test_superset_ingest(pytestconfig, tmp_path, mock_time, requests_mock):
def test_superset_stateful_ingest(
pytestconfig, tmp_path, mock_time, requests_mock, mock_datahub_graph
):
test_resources_dir = pytestconfig.rootpath / "tests/integration/superset"
register_mock_api(request_mock=requests_mock)
@ -241,6 +260,18 @@ def test_superset_stateful_ingest(
"dashboard_title": "test_dashboard_title_1",
"url": "/dashboard/test_dashboard_url_1",
"position_json": '{"CHART-test-1": {"meta": { "chartId": "10" }}, "CHART-test-2": {"meta": { "chartId": "11" }}}',
"status": "published",
"published": True,
"owners": [
{
"username": "test_username_1",
},
{
"username": "test_username_2",
},
],
"certified_by": "Certification team",
"certification_details": "Approved",
},
],
},