diff --git a/metadata-ingestion/src/datahub/ingestion/source/superset.py b/metadata-ingestion/src/datahub/ingestion/source/superset.py index 7f607666db..931069a921 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/superset.py +++ b/metadata-ingestion/src/datahub/ingestion/source/superset.py @@ -267,13 +267,35 @@ class SupersetSource(StatefulIngestionSourceBase): f"urn:li:chart:({self.platform},{value.get('meta', {}).get('chartId', 'unknown')})" ) + # Build properties + custom_properties = { + "Status": str(dashboard_data.get("status")), + "IsPublished": str(dashboard_data.get("published", False)).lower(), + "Owners": ", ".join( + map( + lambda owner: owner.get("username", "unknown"), + dashboard_data.get("owners", []), + ) + ), + "IsCertified": str( + True if dashboard_data.get("certified_by") else False + ).lower(), + } + + if dashboard_data.get("certified_by"): + custom_properties["CertifiedBy"] = dashboard_data.get("certified_by") + custom_properties["CertificationDetails"] = str( + dashboard_data.get("certification_details") + ) + + # Create DashboardInfo object dashboard_info = DashboardInfoClass( description="", title=title, charts=chart_urns, lastModified=last_modified, dashboardUrl=dashboard_url, - customProperties={}, + customProperties=custom_properties, ) dashboard_snapshot.aspects.append(dashboard_info) return dashboard_snapshot diff --git a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json index 6a522281f1..74312940f0 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_ingest.json @@ -11,7 +11,14 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, + "customProperties": { + "Status": "published", + "IsPublished": "true", + "Owners": "test_username_1, test_username_2", + "IsCertified": "true", + "CertifiedBy": "Certification team", + "CertificationDetails": "Approved" + }, "title": "test_dashboard_title_1", "description": "", "charts": [ @@ -52,7 +59,12 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, + "customProperties": { + "Status": "draft", + "IsPublished": "false", + "Owners": "unknown", + "IsCertified": "false" + }, "title": "test_dashboard_title_2", "description": "", "charts": [ diff --git a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json index 268fa37396..cf38341085 100644 --- a/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json +++ b/metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json @@ -11,7 +11,14 @@ }, { "com.linkedin.pegasus2avro.dashboard.DashboardInfo": { - "customProperties": {}, + "customProperties": { + "Status": "published", + "IsPublished": "true", + "Owners": "test_username_1, test_username_2", + "IsCertified": "true", + "CertifiedBy": "Certification team", + "CertificationDetails": "Approved" + }, "title": "test_dashboard_title_1", "description": "", "charts": [ diff --git a/metadata-ingestion/tests/integration/superset/test_superset.py b/metadata-ingestion/tests/integration/superset/test_superset.py index bc299e3651..b3b5982016 100644 --- a/metadata-ingestion/tests/integration/superset/test_superset.py +++ b/metadata-ingestion/tests/integration/superset/test_superset.py @@ -41,6 +41,18 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "dashboard_title": "test_dashboard_title_1", "url": "/dashboard/test_dashboard_url_1", "position_json": '{"CHART-test-1": {"meta": { "chartId": "10" }}, "CHART-test-2": {"meta": { "chartId": "11" }}}', + "status": "published", + "published": True, + "owners": [ + { + "username": "test_username_1", + }, + { + "username": "test_username_2", + }, + ], + "certified_by": "Certification team", + "certification_details": "Approved", }, { "id": "2", @@ -51,6 +63,15 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: "dashboard_title": "test_dashboard_title_2", "url": "/dashboard/test_dashboard_url_2", "position_json": '{"CHART-test-3": {"meta": { "chartId": "12" }}, "CHART-test-4": {"meta": { "chartId": "13" }}}', + "status": "draft", + "published": False, + "owners": [ + { + "first_name": "name", + }, + ], + "certified_by": "", + "certification_details": "", }, ], }, @@ -151,7 +172,6 @@ def register_mock_api(request_mock: Any, override_data: dict = {}) -> None: @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_superset_ingest(pytestconfig, tmp_path, mock_time, requests_mock): - test_resources_dir = pytestconfig.rootpath / "tests/integration/superset" register_mock_api(request_mock=requests_mock) @@ -193,7 +213,6 @@ def test_superset_ingest(pytestconfig, tmp_path, mock_time, requests_mock): def test_superset_stateful_ingest( pytestconfig, tmp_path, mock_time, requests_mock, mock_datahub_graph ): - test_resources_dir = pytestconfig.rootpath / "tests/integration/superset" register_mock_api(request_mock=requests_mock) @@ -241,6 +260,18 @@ def test_superset_stateful_ingest( "dashboard_title": "test_dashboard_title_1", "url": "/dashboard/test_dashboard_url_1", "position_json": '{"CHART-test-1": {"meta": { "chartId": "10" }}, "CHART-test-2": {"meta": { "chartId": "11" }}}', + "status": "published", + "published": True, + "owners": [ + { + "username": "test_username_1", + }, + { + "username": "test_username_2", + }, + ], + "certified_by": "Certification team", + "certification_details": "Approved", }, ], },