Domo issue (#10080)

* Domo issue * added pydantic models and changes as per comment * Fix: added ownership details * fixed pytest
2025-08-24 08:58:06 +00:00 · 2023-02-07 17:16:26 +05:30 · 2023-02-07 17:16:26 +05:30 · 012f96a7a2
commit 012f96a7a2
parent b75573386b
4 changed files with 1022 additions and 875 deletions
--- a/ingestion/src/metadata/clients/domo_client.py
+++ b/ingestion/src/metadata/clients/domo_client.py
@ -14,7 +14,9 @@ DomoClient source to extract data from DOMO
 """

 import traceback
-from typing import Union
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Extra

 from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import (
    DomoDashboardConnection,
@ -34,6 +36,58 @@ HEADERS = {"Content-Type": "application/json"}
 WORKFLOW_URL = "dataprocessing/v1/dataflows"


+class DomoBaseModel(BaseModel):
+    """
+    Domo basic configurations
+    """
+
+    class Config:
+        extra = Extra.allow
+
+    id: str
+    name: str
+
+
+class DomoOwner(BaseModel):
+    """
+    Owner Owner Details
+    """
+
+    displayName: str
+    id: str
+
+
+class DomoDashboardDetails(DomoBaseModel):
+    """
+    Response from Domo API
+    """
+
+    cardIds: Optional[List[int]]
+    collectionIds: Optional[List[int]]
+    description: Optional[str]
+    owners: Optional[List[DomoOwner]]
+
+
+class DomoChartMetadataDetails(BaseModel):
+    """
+    Metadata Details in chart
+    """
+
+    class Config:
+        extra = Extra.allow
+
+    chartType: Optional[str]
+
+
+class DomoChartDetails(DomoBaseModel):
+    """
+    Response from Domo API for chart
+    """
+
+    metadata: DomoChartMetadataDetails
+    description: Optional[str]
+
+
 class DomoClient:
    """
    Implements the necessary methods to extract
@ -47,6 +101,11 @@ class DomoClient:
        ],
    ):
        self.config = config
+        self.config.sandboxDomain = (
+            self.config.sandboxDomain[:-1]
+            if self.config.sandboxDomain.endswith("/")
+            else self.config.sandboxDomain
+        )
        HEADERS.update({"X-DOMO-Developer-Token": self.config.accessToken})
        client_config: ClientConfig = ClientConfig(
            base_url=self.config.sandboxDomain,
@ -56,7 +115,10 @@ class DomoClient:
        )
        self.client = REST(client_config)

-    def get_chart_details(self, page_id) -> dict:
+    def get_chart_details(self, page_id) -> Optional[DomoChartDetails]:
+        """
+        Getting chart details for particular page
+        """
        url = (
            f"content/v1/cards?urns={page_id}&parts=datasources,dateInfo,library,masonData,metadata,"
            f"metadataOverrides,owners,problems,properties,slicers,subscriptions&includeFiltered=true"
@ -65,23 +127,46 @@ class DomoClient:
            response = self.client._request(  # pylint: disable=protected-access
                method="GET", path=url, headers=HEADERS
            )
-            return response[0] if len(response) > 0 else None
+
+            if isinstance(response, list) and len(response) > 0:
+                return DomoChartDetails(
+                    id=response[0]["id"],
+                    name=response[0]["title"],
+                    metadata=DomoChartMetadataDetails(
+                        chartType=response[0].get("metadata", {}).get("chartType", "")
+                    ),
+                    description=response[0].get("description", ""),
+                )

        except Exception as exc:
-            logger.info(f"Error while getting details for Card {page_id} - {exc}")
+            logger.warning(f"Error while getting details for Card {page_id} - {exc}")
            logger.debug(traceback.format_exc())

        return None

+    # def get_owner_details(self, owner_id) -> dict:
+
    def get_pipelines(self):
+        try:
            response = self.client._request(  # pylint: disable=protected-access
                method="GET", path=WORKFLOW_URL, headers=HEADERS
            )
            return response
+        except Exception as exc:
+            logger.warning(f"Error while getting pipelines - {exc}")
+            logger.debug(traceback.format_exc())
+        return []

    def get_runs(self, workflow_id):
+        try:
            url = f"dataprocessing/v1/dataflows/{workflow_id}/executions?limit=100&offset=0"
            response = self.client._request(  # pylint: disable=protected-access
                method="GET", path=url, headers=HEADERS
            )
            return response
+        except Exception as exc:
+            logger.warning(
+                f"Error while getting runs for pipeline {workflow_id} - {exc}"
+            )
+            logger.debug(traceback.format_exc())
+        return []
--- a/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py
+++ b/ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py
@ -17,7 +17,12 @@ from typing import Any, Iterable, List, Optional

 from pydantic import ValidationError

-from metadata.clients.domo_client import DomoClient
+from metadata.clients.domo_client import (
+    DomoChartDetails,
+    DomoClient,
+    DomoDashboardDetails,
+    DomoOwner,
+)
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@ -47,7 +52,7 @@ class DomodashboardSource(DashboardServiceSource):
    """

    config: WorkflowSource
-    metadata: OpenMetadataConnection
+    metadata_config: OpenMetadataConnection
    status: SourceStatus

    def __init__(self, config: WorkflowSource, metadata_config: OpenMetadataConnection):
@ -64,31 +69,59 @@ class DomodashboardSource(DashboardServiceSource):
            )
        return cls(config, metadata_config)

-    def get_dashboards_list(self) -> Optional[List[dict]]:
+    def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]:
        dashboards = self.client.page_list()
        dashboard_list = []
        for dashboard in dashboards:
            dashboard_detail = self.get_page_details(page_id=dashboard["id"])
-            dashboard_list.append(dashboard_detail)
+            dashboard_list.append(
+                DomoDashboardDetails(
+                    id=dashboard_detail.id,
+                    name=dashboard_detail.name,
+                    cardIds=dashboard_detail.cardIds,
+                    description=dashboard_detail.description,
+                    owners=dashboard_detail.owners,
+                    collectionIds=dashboard_detail.collectionIds,
+                )
+            )
        return dashboard_list

-    def get_dashboard_name(self, dashboard: Any) -> str:
-        return dashboard["name"]
+    def get_dashboard_name(self, dashboard: DomoDashboardDetails) -> str:
+        return dashboard.name

-    def get_dashboard_details(self, dashboard: dict) -> dict:
+    def get_dashboard_details(self, dashboard: DomoDashboardDetails) -> dict:
        return dashboard

+    def get_owner_details(self, owners: List[DomoOwner]) -> Optional[EntityReference]:
+        for owner in owners:
+            try:
+                owner_details = self.client.users_get(owner.id)
+                if owner_details.get("email"):
+                    user = self.metadata.get_user_by_email(owner_details["email"])
+                    if user:
+                        return EntityReference(id=user["id"], type="user")
+                    logger.debug(
+                        f"No user for found for email {owner_details['email']} in OMD"
+                    )
+            except Exception as exc:
+                logger.warning(
+                    f"Error while getting details of user {owner.displayName} - {exc}"
+                )
+        return None
+
    def yield_dashboard(
-        self, dashboard_details: dict
+        self, dashboard_details: DomoDashboardDetails
    ) -> Iterable[CreateDashboardRequest]:
        try:
-            dashboard_url = f"{self.service_connection.sandboxDomain}/page/{dashboard_details['id']}"
+            dashboard_url = (
+                f"{self.service_connection.sandboxDomain}/page/{dashboard_details.id}"
+            )

            yield CreateDashboardRequest(
-                name=dashboard_details["id"],
+                name=dashboard_details.id,
                dashboardUrl=dashboard_url,
-                displayName=dashboard_details.get("name"),
-                description=dashboard_details.get("description", ""),
+                displayName=dashboard_details.name,
+                description=dashboard_details.description,
                charts=[
                    EntityReference(id=chart.id.__root__, type="chart")
                    for chart in self.context.charts
@ -97,81 +130,94 @@ class DomodashboardSource(DashboardServiceSource):
                    id=self.context.dashboard_service.id.__root__,
                    type="dashboardService",
                ),
+                owner=self.get_owner_details(dashboard_details.owners),
            )
        except KeyError as err:
            logger.warning(
-                f"Error extracting data from {dashboard_details.get('name', 'unknown')} - {err}"
+                f"Error extracting data from {dashboard_details.name} - {err}"
            )
            logger.debug(traceback.format_exc())
        except ValidationError as err:
            logger.warning(
-                f"Error building pydantic model for {dashboard_details.get('name', 'unknown')} - {err}"
+                f"Error building pydantic model for {dashboard_details.name} - {err}"
            )
            logger.debug(traceback.format_exc())
        except Exception as err:
            logger.warning(
-                f"Wild error ingesting dashboard {dashboard_details.get('name', 'unknown')} - {err}"
+                f"Wild error ingesting dashboard {dashboard_details.name} - {err}"
            )
            logger.debug(traceback.format_exc())

-    def get_page_details(self, page_id):
+    def get_owners(self, owners: List[dict]) -> List[DomoOwner]:
+        domo_owner = []
+        for owner in owners:
+            domo_owner.append(
+                DomoOwner(id=str(owner["id"]), displayName=owner["displayName"])
+            )
+
+        return domo_owner
+
+    def get_page_details(self, page_id) -> Optional[DomoDashboardDetails]:
        try:
            pages = self.client.page_get(page_id)
-            return pages
+            return DomoDashboardDetails(
+                name=pages["name"],
+                id=pages["id"],
+                cardIds=pages.get("cardIds", []),
+                description=pages.get("description", ""),
+                collectionIds=pages.get("collectionIds", []),
+                owners=self.get_owners(pages.get("owners", [])),
+            )
        except Exception as exc:
            logger.warning(
-                f"Error while getting details from collection {page_id} - {exc}"
+                f"Error while getting details from collection page {page_id} - {exc}"
            )
            logger.debug(traceback.format_exc())
            return None

    def get_chart_ids(self, collection_ids: List[Any]):
        chart_ids = []
-        for collection_id in collection_ids:
+        for collection_id in collection_ids or []:
            chart_id = self.get_page_details(page_id=collection_id)
-            for chart in chart_id.get("cardIds", []):
+            for chart in chart_id.cardIds:
                chart_ids.append(chart)
        return chart_ids

    def yield_dashboard_chart(
-        self, dashboard_details: Any
+        self, dashboard_details: DomoDashboardDetails
    ) -> Optional[Iterable[CreateChartRequest]]:
-        chart_ids = dashboard_details["cardIds"]
-        chart_id_from_collection = self.get_chart_ids(
-            dashboard_details.get("collectionIds", [])
-        )
+        chart_ids = dashboard_details.cardIds
+        chart_id_from_collection = self.get_chart_ids(dashboard_details.collectionIds)
        chart_ids.extend(chart_id_from_collection)
        for chart_id in chart_ids:
            try:
-                chart = self.domo_client.get_chart_details(page_id=chart_id)
-
+                chart: DomoChartDetails = self.domo_client.get_chart_details(
+                    page_id=chart_id
+                )
                chart_url = (
                    f"{self.service_connection.sandboxDomain}/page/"
-                    f"{dashboard_details['id']}/kpis/details/{chart_id}"
+                    f"{dashboard_details.id}/kpis/details/{chart_id}"
                )

-                if filter_by_chart(
-                    self.source_config.chartFilterPattern, chart["title"]
-                ):
-                    self.status.filter(chart["title"], "Chart Pattern not allowed")
+                if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
+                    self.status.filter(chart.name, "Chart Pattern not allowed")
                    continue
-
+                if chart.name:
                    yield CreateChartRequest(
                        name=chart_id,
-                    description=chart.get("description", ""),
-                    displayName=chart.get("title"),
-                    chartType=get_standard_chart_type(
-                        chart["metadata"].get("chartType", "")
-                    ).value,
+                        description=chart.description,
+                        displayName=chart.name,
                        chartUrl=chart_url,
                        service=EntityReference(
                            id=self.context.dashboard_service.id.__root__,
                            type="dashboardService",
                        ),
+                        chartType=get_standard_chart_type(chart.metadata.chartType),
                    )
-                self.status.scanned(chart["title"])
+                    self.status.scanned(chart.name)
            except Exception as exc:
                logger.warning(f"Error creating chart [{chart}]: {exc}")
+                self.status.failures.append(f"{dashboard_details.name}.{chart_id}")
                logger.debug(traceback.format_exc())
                continue

--- a/ingestion/tests/unit/resources/datasets/domodashboard_dataset.json
+++ b/ingestion/tests/unit/resources/datasets/domodashboard_dataset.json
@ -1,3 +1,5 @@
+[
+   [
      {
         "id":"552315335",
         "page":{
@ -23,6 +25,7 @@
         },
         "type":"page",
         "title":"New Dashboard",
+         "description": "TOP SALESPEOPLE\nDisplays the top 10 salespeople by won revenue. Identify over-performers and understand the secrets to their success.",
         "sizes":[
            {
               "id":"1108771657",
@ -768,3 +771,7 @@
            "showFilterIcons":true
         }
      }
+   ],
+   "Reponse[301]",
+   {}
+]
--- a/ingestion/tests/unit/topology/dashboard/test_domodashboard.py
+++ b/ingestion/tests/unit/topology/dashboard/test_domodashboard.py
@ -19,7 +19,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
    OpenMetadataWorkflowConfig,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.ometa.client import REST
 from metadata.ingestion.source.dashboard.domodashboard.metadata import (
+    DomoDashboardDetails,
    DomodashboardSource,
 )

@ -83,16 +85,18 @@ mock_domopipeline_config = {
    },
 }

-MOCK_DASHBOARD = {
-    "id": 552315335,
-    "name": "New Dashboard",
-    "children": [],
-    "cardIds": ["1982511286", "781210736"],
-}
+MOCK_DASHBOARD = DomoDashboardDetails(
+    id=552315335,
+    name="New Dashboard",
+    cardIds=["1982511286", "781210736"],
+    collection_ids=[],
+    owners=[],
+)
+
 EXPECTED_DASHBOARD = CreateDashboardRequest(
    name="552315335",
    displayName="New Dashboard",
-    description="",
+    description=None,
    dashboardUrl="https://domain.domo.com/page/552315335",
    charts=[],
    tags=None,
@ -110,16 +114,16 @@ EXPECTED_DASHBOARD = CreateDashboardRequest(
    extension=None,
 )

-EXPECTED_DASHBOARDS = [
+EXPECTED_CHARTS = [
    CreateChartRequest(
-        name="Top Salespeople",
-        displayName="Top Salespeople",
+        name="1982511286",
+        displayName="New Dashboard",
        description=(
            "TOP SALESPEOPLE\nDisplays the top 10 salespeople by won revenue."
            " Identify over-performers and understand the secrets to their success."
        ),
        chartType="Other",
-        chartUrl="https://domain.domo.com/page/552315335/kpis/details/1108771657",
+        chartUrl="https://domain.domo.com/page/552315335/kpis/details/1982511286",
        tables=None,
        tags=None,
        owner=None,
@ -135,31 +139,14 @@ EXPECTED_DASHBOARDS = [
        ),
    ),
    CreateChartRequest(
-        name="Milan Datasets",
-        displayName="Milan Datasets",
-        description="",
-        chartType="Other",
-        chartUrl="https://domain.domo.com/page/552315335/kpis/details/1985861713",
-        tables=None,
-        tags=None,
-        owner=None,
-        service=EntityReference(
-            id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb",
-            type="dashboardService",
-            name=None,
-            fullyQualifiedName=None,
-            description=None,
-            displayName=None,
-            deleted=None,
-            href=None,
+        name="781210736",
+        displayName="New Dashboard",
+        description=(
+            "TOP SALESPEOPLE\nDisplays the top 10 salespeople by won revenue."
+            " Identify over-performers and understand the secrets to their success."
        ),
-    ),
-    CreateChartRequest(
-        name="Page Fans",
-        displayName="Page Fans",
-        description="",
        chartType="Other",
-        chartUrl="https://domain.domo.com/page/552315335/kpis/details/2025899139",
+        chartUrl="https://domain.domo.com/page/552315335/kpis/details/781210736",
        tables=None,
        tags=None,
        owner=None,
@ -211,16 +198,38 @@ class DomoDashboardUnitTest(TestCase):

    def test_dashboard_name(self):
        assert (
-            self.domodashboard.get_dashboard_name(MOCK_DASHBOARD) == mock_data["title"]
+            self.domodashboard.get_dashboard_name(MOCK_DASHBOARD)
+            == mock_data[0][0]["title"]
        )

-    @patch("metadata.clients.domo_client.DomoClient.get_chart_details")
-    def test_chart(self, get_chart_details):
-        get_chart_details.return_value = mock_data
+    def test_chart(self):
+        """
+        Function for testing charts
+        """
+        with patch.object(REST, "_request", return_value=mock_data[0]):
            results = self.domodashboard.yield_dashboard_chart(MOCK_DASHBOARD)
            chart_list = []
            for result in results:
                if isinstance(result, CreateChartRequest):
                    chart_list.append(result)
-        for _, (expected, original) in enumerate(zip(EXPECTED_DASHBOARDS, chart_list)):
+            for _, (expected, original) in enumerate(zip(EXPECTED_CHARTS, chart_list)):
                self.assertEqual(expected, original)
+
+        with patch.object(REST, "_request", return_value=mock_data[1]):
+            result = self.domodashboard.domo_client.get_chart_details(
+                MOCK_DASHBOARD.cardIds[0]
+            )
+            assert (
+                self.domodashboard.domo_client.get_chart_details(
+                    MOCK_DASHBOARD.cardIds[0]
+                )
+                is None
+            )
+
+        with patch.object(REST, "_request", return_value=mock_data[2]):
+            assert (
+                self.domodashboard.domo_client.get_chart_details(
+                    MOCK_DASHBOARD.cardIds[0]
+                )
+                is None
+            )