Domo issue (#10080)

* Domo issue

* added pydantic models and changes as per comment

* Fix: added ownership details

* fixed pytest

ingestion/src/metadata/clients/domo_client.py

@@ -14,7 +14,9 @@ DomoClient source to extract data from DOMO
 """
 
 import traceback
-from typing import Union
+from typing import List, Optional, Union
+
+from pydantic import BaseModel, Extra
 
 from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import (
     DomoDashboardConnection,
@@ -34,6 +36,58 @@ HEADERS = {"Content-Type": "application/json"}
 WORKFLOW_URL = "dataprocessing/v1/dataflows"
 
 
+class DomoBaseModel(BaseModel):
+    """
+    Domo basic configurations
+    """
+
+    class Config:
+        extra = Extra.allow
+
+    id: str
+    name: str
+
+
+class DomoOwner(BaseModel):
+    """
+    Owner Owner Details
+    """
+
+    displayName: str
+    id: str
+
+
+class DomoDashboardDetails(DomoBaseModel):
+    """
+    Response from Domo API
+    """
+
+    cardIds: Optional[List[int]]
+    collectionIds: Optional[List[int]]
+    description: Optional[str]
+    owners: Optional[List[DomoOwner]]
+
+
+class DomoChartMetadataDetails(BaseModel):
+    """
+    Metadata Details in chart
+    """
+
+    class Config:
+        extra = Extra.allow
+
+    chartType: Optional[str]
+
+
+class DomoChartDetails(DomoBaseModel):
+    """
+    Response from Domo API for chart
+    """
+
+    metadata: DomoChartMetadataDetails
+    description: Optional[str]
+
+
 class DomoClient:
     """
     Implements the necessary methods to extract
@@ -47,6 +101,11 @@ class DomoClient:
         ],
     ):
         self.config = config
+        self.config.sandboxDomain = (
+            self.config.sandboxDomain[:-1]
+            if self.config.sandboxDomain.endswith("/")
+            else self.config.sandboxDomain
+        )
         HEADERS.update({"X-DOMO-Developer-Token": self.config.accessToken})
         client_config: ClientConfig = ClientConfig(
             base_url=self.config.sandboxDomain,
@@ -56,7 +115,10 @@ class DomoClient:
         )
         self.client = REST(client_config)
 
-    def get_chart_details(self, page_id) -> dict:
+    def get_chart_details(self, page_id) -> Optional[DomoChartDetails]:
+        """
+        Getting chart details for particular page
+        """
         url = (
             f"content/v1/cards?urns={page_id}&parts=datasources,dateInfo,library,masonData,metadata,"
             f"metadataOverrides,owners,problems,properties,slicers,subscriptions&includeFiltered=true"
@@ -65,23 +127,46 @@ class DomoClient:
             response = self.client._request(  # pylint: disable=protected-access
                 method="GET", path=url, headers=HEADERS
             )
-            return response[0] if len(response) > 0 else None
+
+            if isinstance(response, list) and len(response) > 0:
+                return DomoChartDetails(
+                    id=response[0]["id"],
+                    name=response[0]["title"],
+                    metadata=DomoChartMetadataDetails(
+                        chartType=response[0].get("metadata", {}).get("chartType", "")
+                    ),
+                    description=response[0].get("description", ""),
+                )
 
         except Exception as exc:
-            logger.info(f"Error while getting details for Card {page_id} - {exc}")
+            logger.warning(f"Error while getting details for Card {page_id} - {exc}")
             logger.debug(traceback.format_exc())
 
         return None
 
+    # def get_owner_details(self, owner_id) -> dict:
+
     def get_pipelines(self):
-        response = self.client._request(  # pylint: disable=protected-access
+        try:
-            method="GET", path=WORKFLOW_URL, headers=HEADERS
+            response = self.client._request(  # pylint: disable=protected-access
-        )
+                method="GET", path=WORKFLOW_URL, headers=HEADERS
-        return response
+            )
+            return response
+        except Exception as exc:
+            logger.warning(f"Error while getting pipelines - {exc}")
+            logger.debug(traceback.format_exc())
+        return []
 
     def get_runs(self, workflow_id):
-        url = f"dataprocessing/v1/dataflows/{workflow_id}/executions?limit=100&offset=0"
+        try:
-        response = self.client._request(  # pylint: disable=protected-access
+            url = f"dataprocessing/v1/dataflows/{workflow_id}/executions?limit=100&offset=0"
-            method="GET", path=url, headers=HEADERS
+            response = self.client._request(  # pylint: disable=protected-access
-        )
+                method="GET", path=url, headers=HEADERS
-        return response
+            )
+            return response
+        except Exception as exc:
+            logger.warning(
+                f"Error while getting runs for pipeline {workflow_id} - {exc}"
+            )
+            logger.debug(traceback.format_exc())
+        return []

ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -17,7 +17,12 @@ from typing import Any, Iterable, List, Optional
 
 from pydantic import ValidationError
 
-from metadata.clients.domo_client import DomoClient
+from metadata.clients.domo_client import (
+    DomoChartDetails,
+    DomoClient,
+    DomoDashboardDetails,
+    DomoOwner,
+)
 from metadata.generated.schema.api.data.createChart import CreateChartRequest
 from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
 from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
@@ -47,7 +52,7 @@ class DomodashboardSource(DashboardServiceSource):
     """
 
     config: WorkflowSource
-    metadata: OpenMetadataConnection
+    metadata_config: OpenMetadataConnection
     status: SourceStatus
 
     def __init__(self, config: WorkflowSource, metadata_config: OpenMetadataConnection):
@@ -64,31 +69,59 @@ class DomodashboardSource(DashboardServiceSource):
             )
         return cls(config, metadata_config)
 
-    def get_dashboards_list(self) -> Optional[List[dict]]:
+    def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]:
         dashboards = self.client.page_list()
         dashboard_list = []
         for dashboard in dashboards:
             dashboard_detail = self.get_page_details(page_id=dashboard["id"])
-            dashboard_list.append(dashboard_detail)
+            dashboard_list.append(
+                DomoDashboardDetails(
+                    id=dashboard_detail.id,
+                    name=dashboard_detail.name,
+                    cardIds=dashboard_detail.cardIds,
+                    description=dashboard_detail.description,
+                    owners=dashboard_detail.owners,
+                    collectionIds=dashboard_detail.collectionIds,
+                )
+            )
         return dashboard_list
 
-    def get_dashboard_name(self, dashboard: Any) -> str:
+    def get_dashboard_name(self, dashboard: DomoDashboardDetails) -> str:
-        return dashboard["name"]
+        return dashboard.name
 
-    def get_dashboard_details(self, dashboard: dict) -> dict:
+    def get_dashboard_details(self, dashboard: DomoDashboardDetails) -> dict:
         return dashboard
 
+    def get_owner_details(self, owners: List[DomoOwner]) -> Optional[EntityReference]:
+        for owner in owners:
+            try:
+                owner_details = self.client.users_get(owner.id)
+                if owner_details.get("email"):
+                    user = self.metadata.get_user_by_email(owner_details["email"])
+                    if user:
+                        return EntityReference(id=user["id"], type="user")
+                    logger.debug(
+                        f"No user for found for email {owner_details['email']} in OMD"
+                    )
+            except Exception as exc:
+                logger.warning(
+                    f"Error while getting details of user {owner.displayName} - {exc}"
+                )
+        return None
+
     def yield_dashboard(
-        self, dashboard_details: dict
+        self, dashboard_details: DomoDashboardDetails
     ) -> Iterable[CreateDashboardRequest]:
         try:
-            dashboard_url = f"{self.service_connection.sandboxDomain}/page/{dashboard_details['id']}"
+            dashboard_url = (
+                f"{self.service_connection.sandboxDomain}/page/{dashboard_details.id}"
+            )
 
             yield CreateDashboardRequest(
-                name=dashboard_details["id"],
+                name=dashboard_details.id,
                 dashboardUrl=dashboard_url,
-                displayName=dashboard_details.get("name"),
+                displayName=dashboard_details.name,
-                description=dashboard_details.get("description", ""),
+                description=dashboard_details.description,
                 charts=[
                     EntityReference(id=chart.id.__root__, type="chart")
                     for chart in self.context.charts
@@ -97,81 +130,94 @@ class DomodashboardSource(DashboardServiceSource):
                     id=self.context.dashboard_service.id.__root__,
                     type="dashboardService",
                 ),
+                owner=self.get_owner_details(dashboard_details.owners),
             )
         except KeyError as err:
             logger.warning(
-                f"Error extracting data from {dashboard_details.get('name', 'unknown')} - {err}"
+                f"Error extracting data from {dashboard_details.name} - {err}"
             )
             logger.debug(traceback.format_exc())
         except ValidationError as err:
             logger.warning(
-                f"Error building pydantic model for {dashboard_details.get('name', 'unknown')} - {err}"
+                f"Error building pydantic model for {dashboard_details.name} - {err}"
             )
             logger.debug(traceback.format_exc())
         except Exception as err:
             logger.warning(
-                f"Wild error ingesting dashboard {dashboard_details.get('name', 'unknown')} - {err}"
+                f"Wild error ingesting dashboard {dashboard_details.name} - {err}"
             )
             logger.debug(traceback.format_exc())
 
-    def get_page_details(self, page_id):
+    def get_owners(self, owners: List[dict]) -> List[DomoOwner]:
+        domo_owner = []
+        for owner in owners:
+            domo_owner.append(
+                DomoOwner(id=str(owner["id"]), displayName=owner["displayName"])
+            )
+
+        return domo_owner
+
+    def get_page_details(self, page_id) -> Optional[DomoDashboardDetails]:
         try:
             pages = self.client.page_get(page_id)
-            return pages
+            return DomoDashboardDetails(
+                name=pages["name"],
+                id=pages["id"],
+                cardIds=pages.get("cardIds", []),
+                description=pages.get("description", ""),
+                collectionIds=pages.get("collectionIds", []),
+                owners=self.get_owners(pages.get("owners", [])),
+            )
         except Exception as exc:
             logger.warning(
-                f"Error while getting details from collection {page_id} - {exc}"
+                f"Error while getting details from collection page {page_id} - {exc}"
             )
             logger.debug(traceback.format_exc())
             return None
 
     def get_chart_ids(self, collection_ids: List[Any]):
         chart_ids = []
-        for collection_id in collection_ids:
+        for collection_id in collection_ids or []:
             chart_id = self.get_page_details(page_id=collection_id)
-            for chart in chart_id.get("cardIds", []):
+            for chart in chart_id.cardIds:
                 chart_ids.append(chart)
         return chart_ids
 
     def yield_dashboard_chart(
-        self, dashboard_details: Any
+        self, dashboard_details: DomoDashboardDetails
     ) -> Optional[Iterable[CreateChartRequest]]:
-        chart_ids = dashboard_details["cardIds"]
+        chart_ids = dashboard_details.cardIds
-        chart_id_from_collection = self.get_chart_ids(
+        chart_id_from_collection = self.get_chart_ids(dashboard_details.collectionIds)
-            dashboard_details.get("collectionIds", [])
-        )
         chart_ids.extend(chart_id_from_collection)
         for chart_id in chart_ids:
             try:
-                chart = self.domo_client.get_chart_details(page_id=chart_id)
+                chart: DomoChartDetails = self.domo_client.get_chart_details(
-
+                    page_id=chart_id
+                )
                 chart_url = (
                     f"{self.service_connection.sandboxDomain}/page/"
-                    f"{dashboard_details['id']}/kpis/details/{chart_id}"
+                    f"{dashboard_details.id}/kpis/details/{chart_id}"
                 )
 
-                if filter_by_chart(
+                if filter_by_chart(self.source_config.chartFilterPattern, chart.name):
-                    self.source_config.chartFilterPattern, chart["title"]
+                    self.status.filter(chart.name, "Chart Pattern not allowed")
-                ):
-                    self.status.filter(chart["title"], "Chart Pattern not allowed")
                     continue
-
+                if chart.name:
-                yield CreateChartRequest(
+                    yield CreateChartRequest(
-                    name=chart_id,
+                        name=chart_id,
-                    description=chart.get("description", ""),
+                        description=chart.description,
-                    displayName=chart.get("title"),
+                        displayName=chart.name,
-                    chartType=get_standard_chart_type(
+                        chartUrl=chart_url,
-                        chart["metadata"].get("chartType", "")
+                        service=EntityReference(
-                    ).value,
+                            id=self.context.dashboard_service.id.__root__,
-                    chartUrl=chart_url,
+                            type="dashboardService",
-                    service=EntityReference(
+                        ),
-                        id=self.context.dashboard_service.id.__root__,
+                        chartType=get_standard_chart_type(chart.metadata.chartType),
-                        type="dashboardService",
+                    )
-                    ),
+                    self.status.scanned(chart.name)
-                )
-                self.status.scanned(chart["title"])
             except Exception as exc:
                 logger.warning(f"Error creating chart [{chart}]: {exc}")
+                self.status.failures.append(f"{dashboard_details.name}.{chart_id}")
                 logger.debug(traceback.format_exc())
                 continue
 

ingestion/tests/unit/topology/dashboard/test_domodashboard.py

@@ -19,7 +19,9 @@ from metadata.generated.schema.metadataIngestion.workflow import (
     OpenMetadataWorkflowConfig,
 )
 from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.ingestion.ometa.client import REST
 from metadata.ingestion.source.dashboard.domodashboard.metadata import (
+    DomoDashboardDetails,
     DomodashboardSource,
 )
 
@@ -83,16 +85,18 @@ mock_domopipeline_config = {
     },
 }
 
-MOCK_DASHBOARD = {
+MOCK_DASHBOARD = DomoDashboardDetails(
-    "id": 552315335,
+    id=552315335,
-    "name": "New Dashboard",
+    name="New Dashboard",
-    "children": [],
+    cardIds=["1982511286", "781210736"],
-    "cardIds": ["1982511286", "781210736"],
+    collection_ids=[],
-}
+    owners=[],
+)
+
 EXPECTED_DASHBOARD = CreateDashboardRequest(
     name="552315335",
     displayName="New Dashboard",
-    description="",
+    description=None,
     dashboardUrl="https://domain.domo.com/page/552315335",
     charts=[],
     tags=None,
@@ -110,16 +114,16 @@ EXPECTED_DASHBOARD = CreateDashboardRequest(
     extension=None,
 )
 
-EXPECTED_DASHBOARDS = [
+EXPECTED_CHARTS = [
     CreateChartRequest(
-        name="Top Salespeople",
+        name="1982511286",
-        displayName="Top Salespeople",
+        displayName="New Dashboard",
         description=(
             "TOP SALESPEOPLE\nDisplays the top 10 salespeople by won revenue."
             " Identify over-performers and understand the secrets to their success."
         ),
         chartType="Other",
-        chartUrl="https://domain.domo.com/page/552315335/kpis/details/1108771657",
+        chartUrl="https://domain.domo.com/page/552315335/kpis/details/1982511286",
         tables=None,
         tags=None,
         owner=None,
@@ -135,31 +139,14 @@ EXPECTED_DASHBOARDS = [
         ),
     ),
     CreateChartRequest(
-        name="Milan Datasets",
+        name="781210736",
-        displayName="Milan Datasets",
+        displayName="New Dashboard",
-        description="",
+        description=(
-        chartType="Other",
+            "TOP SALESPEOPLE\nDisplays the top 10 salespeople by won revenue."
-        chartUrl="https://domain.domo.com/page/552315335/kpis/details/1985861713",
+            " Identify over-performers and understand the secrets to their success."
-        tables=None,
-        tags=None,
-        owner=None,
-        service=EntityReference(
-            id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb",
-            type="dashboardService",
-            name=None,
-            fullyQualifiedName=None,
-            description=None,
-            displayName=None,
-            deleted=None,
-            href=None,
         ),
-    ),
-    CreateChartRequest(
-        name="Page Fans",
-        displayName="Page Fans",
-        description="",
         chartType="Other",
-        chartUrl="https://domain.domo.com/page/552315335/kpis/details/2025899139",
+        chartUrl="https://domain.domo.com/page/552315335/kpis/details/781210736",
         tables=None,
         tags=None,
         owner=None,
@@ -211,16 +198,38 @@ class DomoDashboardUnitTest(TestCase):
 
     def test_dashboard_name(self):
         assert (
-            self.domodashboard.get_dashboard_name(MOCK_DASHBOARD) == mock_data["title"]
+            self.domodashboard.get_dashboard_name(MOCK_DASHBOARD)
+            == mock_data[0][0]["title"]
         )
 
-    @patch("metadata.clients.domo_client.DomoClient.get_chart_details")
+    def test_chart(self):
-    def test_chart(self, get_chart_details):
+        """
-        get_chart_details.return_value = mock_data
+        Function for testing charts
-        results = self.domodashboard.yield_dashboard_chart(MOCK_DASHBOARD)
+        """
-        chart_list = []
+        with patch.object(REST, "_request", return_value=mock_data[0]):
-        for result in results:
+            results = self.domodashboard.yield_dashboard_chart(MOCK_DASHBOARD)
-            if isinstance(result, CreateChartRequest):
+            chart_list = []
-                chart_list.append(result)
+            for result in results:
-        for _, (expected, original) in enumerate(zip(EXPECTED_DASHBOARDS, chart_list)):
+                if isinstance(result, CreateChartRequest):
-            self.assertEqual(expected, original)
+                    chart_list.append(result)
+            for _, (expected, original) in enumerate(zip(EXPECTED_CHARTS, chart_list)):
+                self.assertEqual(expected, original)
+
+        with patch.object(REST, "_request", return_value=mock_data[1]):
+            result = self.domodashboard.domo_client.get_chart_details(
+                MOCK_DASHBOARD.cardIds[0]
+            )
+            assert (
+                self.domodashboard.domo_client.get_chart_details(
+                    MOCK_DASHBOARD.cardIds[0]
+                )
+                is None
+            )
+
+        with patch.object(REST, "_request", return_value=mock_data[2]):
+            assert (
+                self.domodashboard.domo_client.get_chart_details(
+                    MOCK_DASHBOARD.cardIds[0]
+                )
+                is None
+            )