Fix #13237 - Rename to instanceDomain and test DomoDashboard charts (#13247)

* Rename sandboxDomain to instanceDomain

* Test Get Charts in DomoDashboard

* Fix schemas

* Fix test

* Fix test

* Rename to Auto Tag PII

* Fix query test

* Fix query test

* Fix query test

bootstrap/sql/migrations/native/1.2.0/mysql/schemaChanges.sql

@@ -85,5 +85,30 @@ SET json = JSON_INSERT(
 )
 WHERE JSON_EXTRACT(json, '$.pipelineType') = 'metadata';
 
+-- Rename sandboxDomain for instanceDomain
+UPDATE dbservice_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.sandboxDomain'),
+    '$.connection.config.instanceDomain',
+    JSON_EXTRACT(json, '$.connection.config.sandboxDomain')
+)
+WHERE serviceType = 'DomoDatabase';
+
+UPDATE dashboard_service_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.sandboxDomain'),
+    '$.connection.config.instanceDomain',
+    JSON_EXTRACT(json, '$.connection.config.sandboxDomain')
+)
+WHERE serviceType = 'DomoDashboard';
+
+UPDATE pipeline_service_entity
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.connection.config.sandboxDomain'),
+    '$.connection.config.instanceDomain',
+    JSON_EXTRACT(json, '$.connection.config.sandboxDomain')
+)
+WHERE serviceType = 'DomoPipeline';
+
 -- Query Entity supports service, which requires FQN for name
 ALTER TABLE query_entity CHANGE COLUMN nameHash fqnHash VARCHAR(256);

bootstrap/sql/migrations/native/1.2.0/postgres/schemaChanges.sql

@@ -102,5 +102,33 @@ SET json = jsonb_set(
 )
 WHERE json #>> '{pipelineType}' = 'metadata';
 
+-- Rename sandboxDomain for instanceDomain
+UPDATE dbservice_entity
+SET json = jsonb_set(
+  json::jsonb #- '{connection,config,sandboxDomain}',
+  '{connection,config,instanceDomain}',
+  (json #> '{connection,config,sandboxDomain}')::jsonb,
+  true
+)
+WHERE serviceType = 'DomoDatabase';
+
+UPDATE dashboard_service_entity
+SET json = jsonb_set(
+  json::jsonb #- '{connection,config,sandboxDomain}',
+  '{connection,config,instanceDomain}',
+  (json #> '{connection,config,sandboxDomain}')::jsonb,
+  true
+)
+WHERE serviceType = 'DomoDashboard';
+
+UPDATE pipeline_service_entity
+SET json = jsonb_set(
+  json::jsonb #- '{connection,config,sandboxDomain}',
+  '{connection,config,instanceDomain}',
+  (json #> '{connection,config,sandboxDomain}')::jsonb,
+  true
+)
+WHERE serviceType = 'DomoPipeline';
+
 -- Query Entity supports service, which requires FQN for name
-ALTER TABLE query_entity RENAME COLUMN nameHash TO fqnHash;
+ALTER TABLE query_entity RENAME COLUMN nameHash TO fqnHash;

ingestion/src/metadata/clients/domo_client.py

@@ -14,9 +14,11 @@ DomoClient source to extract data from DOMO
 """
 
 import traceback
+from dataclasses import dataclass
 from typing import List, Optional, Union
 
 from pydantic import BaseModel, Extra
+from pydomo import Domo
 
 from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import (
     DomoDashboardConnection,
@@ -101,14 +103,14 @@ class DomoClient:
         ],
     ):
         self.config = config
-        self.config.sandboxDomain = (
+        self.config.instanceDomain = (
-            self.config.sandboxDomain[:-1]
+            self.config.instanceDomain[:-1]
-            if self.config.sandboxDomain.endswith("/")
+            if self.config.instanceDomain.endswith("/")
-            else self.config.sandboxDomain
+            else self.config.instanceDomain
         )
         HEADERS.update({"X-DOMO-Developer-Token": self.config.accessToken})
         client_config: ClientConfig = ClientConfig(
-            base_url=self.config.sandboxDomain,
+            base_url=self.config.instanceDomain,
             api_version="api/",
             auth_header="Authorization",
             auth_token=lambda: ("no_token", 0),
@@ -170,3 +172,29 @@ class DomoClient:
             )
             logger.debug(traceback.format_exc())
         return []
+
+    def test_list_cards(self) -> None:
+        """
+        Test function to list the cards. Since we are not passing any URNS from the dashboard
+        we expect an empty result. However, the call should not fail with any 401 error.
+        This helps us validate that the provided Access Token is correct for Domo Dashboard.
+        """
+        try:
+            self.client._request(  # pylint: disable=protected-access
+                method="GET", path="content/v1/cards", headers=HEADERS
+            )
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(f"Error listing cards due to [{exc}]")
+            raise exc
+
+
+@dataclass
+class OMPyDomoClient:
+    """
+    domo_client: official pydomo client https://github.com/domoinc/domo-python-sdk
+    client: custom requests on the instance domain
+    """
+
+    domo: Domo
+    custom: DomoClient

ingestion/src/metadata/examples/workflows/domodashboard.yaml

@@ -8,7 +8,7 @@ source:
       secretToken: secret-token
       accessToken: access-token
       apiHost: api.domo.com
-      sandboxDomain: https://api_domo.domo.com
+      instanceDomain: https://api_domo.domo.com
   sourceConfig:
     config:
       type: DashboardMetadata

ingestion/src/metadata/ingestion/source/dashboard/domodashboard/connection.py

@@ -17,6 +17,7 @@ from typing import Optional
 
 from pydomo import Domo
 
+from metadata.clients.domo_client import DomoClient, OMPyDomoClient
 from metadata.generated.schema.entity.automations.workflow import (
     Workflow as AutomationWorkflow,
 )
@@ -30,7 +31,7 @@ from metadata.ingestion.connections.test_connections import (
 from metadata.ingestion.ometa.ometa_api import OpenMetadata
 
 
-def get_connection(connection: DomoDashboardConnection) -> Domo:
+def get_connection(connection: DomoDashboardConnection) -> OMPyDomoClient:
     """
     Create connection
     """
@@ -40,7 +41,12 @@ def get_connection(connection: DomoDashboardConnection) -> Domo:
             connection.secretToken.get_secret_value(),
             api_host=connection.apiHost,
         )
-        return domo
+        client = DomoClient(connection)
+
+        return OMPyDomoClient(
+            domo=domo,
+            custom=client,
+        )
     except Exception as exc:
         msg = f"Unknown error connecting with {connection}: {exc}."
         raise SourceConnectionException(msg)
@@ -48,7 +54,7 @@ def get_connection(connection: DomoDashboardConnection) -> Domo:
 
 def test_connection(
     metadata: OpenMetadata,
-    domo: Domo,
+    client: OMPyDomoClient,
     service_connection: DomoDashboardConnection,
     automation_workflow: Optional[AutomationWorkflow] = None,
 ) -> None:
@@ -57,11 +63,14 @@ def test_connection(
     of a metadata workflow or during an Automation Workflow
     """
 
-    def custom_executor():
+    def custom_test_page_list():
-        result = domo.page_list()
+        result = client.domo.page_list()
         return list(result)
 
-    test_fn = {"GetDashboards": custom_executor}
+    test_fn = {
+        "GetDashboards": custom_test_page_list,
+        "GetCharts": client.custom.test_list_cards,
+    }
 
     test_connection_steps(
         metadata=metadata,

ingestion/src/metadata/ingestion/source/dashboard/domodashboard/metadata.py

@@ -19,7 +19,6 @@ from pydantic import ValidationError
 
 from metadata.clients.domo_client import (
     DomoChartDetails,
-    DomoClient,
     DomoDashboardDetails,
     DomoOwner,
 )
@@ -57,10 +56,6 @@ class DomodashboardSource(DashboardServiceSource):
     config: WorkflowSource
     metadata_config: OpenMetadataConnection
 
-    def __init__(self, config: WorkflowSource, metadata_config: OpenMetadataConnection):
-        super().__init__(config, metadata_config)
-        self.domo_client = DomoClient(self.service_connection)
-
     @classmethod
     def create(cls, config_dict, metadata_config: OpenMetadataConnection):
         config = WorkflowSource.parse_obj(config_dict)
@@ -72,7 +67,7 @@ class DomodashboardSource(DashboardServiceSource):
         return cls(config, metadata_config)
 
     def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]:
-        dashboards = self.client.page_list()
+        dashboards = self.client.domo.page_list()
         dashboard_list = []
         for dashboard in dashboards:
             dashboard_detail = self.get_page_details(page_id=dashboard["id"])
@@ -99,7 +94,7 @@ class DomodashboardSource(DashboardServiceSource):
     ) -> Optional[EntityReference]:
         for owner in dashboard_details.owners:
             try:
-                owner_details = self.client.users_get(owner.id)
+                owner_details = self.client.domo.users_get(owner.id)
                 if owner_details.get("email"):
                     user = self.metadata.get_user_by_email(owner_details["email"])
                     if user:
@@ -118,7 +113,7 @@ class DomodashboardSource(DashboardServiceSource):
     ) -> Iterable[Either[CreateDashboardRequest]]:
         try:
             dashboard_url = (
-                f"{self.service_connection.sandboxDomain}/page/{dashboard_details.id}"
+                f"{self.service_connection.instanceDomain}/page/{dashboard_details.id}"
             )
 
             dashboard_request = CreateDashboardRequest(
@@ -175,7 +170,7 @@ class DomodashboardSource(DashboardServiceSource):
 
     def get_page_details(self, page_id) -> Optional[DomoDashboardDetails]:
         try:
-            pages = self.client.page_get(page_id)
+            pages = self.client.domo.page_get(page_id)
             return DomoDashboardDetails(
                 name=pages["name"],
                 id=pages["id"],
@@ -208,9 +203,9 @@ class DomodashboardSource(DashboardServiceSource):
         for chart_id in chart_ids:
             chart: Optional[DomoChartDetails] = None
             try:
-                chart = self.domo_client.get_chart_details(page_id=chart_id)
+                chart = self.client.custom.get_chart_details(page_id=chart_id)
                 chart_url = (
-                    f"{self.service_connection.sandboxDomain}/page/"
+                    f"{self.service_connection.instanceDomain}/page/"
                     f"{dashboard_details.id}/kpis/details/{chart_id}"
                 )
 

ingestion/src/metadata/ingestion/source/database/domodatabase/metadata.py

@@ -16,7 +16,6 @@ Domo Database source to extract metadata
 import traceback
 from typing import Any, Iterable, List, Optional, Tuple
 
-from metadata.clients.domo_client import DomoClient
 from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
 from metadata.generated.schema.api.data.createDatabaseSchema import (
     CreateDatabaseSchemaRequest,
@@ -81,7 +80,6 @@ class DomodatabaseSource(DatabaseServiceSource):
         self.service_connection = self.config.serviceConnection.__root__.config
         self.domo_client = get_connection(self.service_connection)
         self.connection_obj = self.domo_client
-        self.client = DomoClient(self.service_connection)
         self.test_connection()
 
     @classmethod
@@ -258,7 +256,7 @@ class DomodatabaseSource(DatabaseServiceSource):
         Method to get the source url for domodatabase
         """
         try:
-            return f"{clean_uri(self.service_connection.sandboxDomain)}/datasources/{table_name}/details/overview"
+            return f"{clean_uri(self.service_connection.instanceDomain)}/datasources/{table_name}/details/overview"
         except Exception as exc:
             logger.debug(traceback.format_exc())
             logger.warning(f"Unable to get source url for {table_name}: {exc}")
@@ -270,4 +268,4 @@ class DomodatabaseSource(DatabaseServiceSource):
         return table
 
     def close(self) -> None:
-        self.client.client.close()
+        """Nothing to close"""

ingestion/src/metadata/ingestion/source/pipeline/domopipeline/metadata.py

@@ -171,7 +171,7 @@ class DomopipelineSource(PipelineServiceSource):
     ) -> Optional[str]:
         try:
             return (
-                f"{clean_uri(self.service_connection.sandboxDomain)}/datacenter/dataflows/"
+                f"{clean_uri(self.service_connection.instanceDomain)}/datacenter/dataflows/"
                 f"{pipeline_id}/details#history"
             )
         except Exception as exc:

ingestion/tests/unit/topology/dashboard/test_domodashboard.py

@@ -62,7 +62,7 @@ mock_domopipeline_config = {
                 "secretToken": "abcdefg",
                 "accessToken": "accessTpokem",
                 "apiHost": "api.domo.com",
-                "sandboxDomain": "https://domain.domo.com",
+                "instanceDomain": "https://domain.domo.com",
             }
         },
         "sourceConfig": {
@@ -187,12 +187,10 @@ class DomoDashboardUnitTest(TestCase):
             for _, (expected, original) in enumerate(zip(EXPECTED_CHARTS, chart_list)):
                 self.assertEqual(expected, original)
 
+        # Cover error responses
         with patch.object(REST, "_request", return_value=mock_data[1]):
-            result = self.domodashboard.domo_client.get_chart_details(
-                MOCK_DASHBOARD.cardIds[0]
-            )
             assert (
-                self.domodashboard.domo_client.get_chart_details(
+                self.domodashboard.client.custom.get_chart_details(
                     MOCK_DASHBOARD.cardIds[0]
                 )
                 is None
@@ -200,7 +198,7 @@ class DomoDashboardUnitTest(TestCase):
 
         with patch.object(REST, "_request", return_value=mock_data[2]):
             assert (
-                self.domodashboard.domo_client.get_chart_details(
+                self.domodashboard.client.custom.get_chart_details(
                     MOCK_DASHBOARD.cardIds[0]
                 )
                 is None

ingestion/tests/unit/topology/database/test_domodatabase.py

@@ -84,7 +84,7 @@ mock_domodatabase_config = {
                 "secretToken": "abcdefg",
                 "accessToken": "accessTpokem",
                 "apiHost": "api.domo.com",
-                "sandboxDomain": "https://domain.domo.com",
+                "instanceDomain": "https://domain.domo.com",
             }
         },
         "sourceConfig": {"config": {"type": "DatabaseMetadata"}},

ingestion/tests/unit/topology/pipeline/test_domopipeline.py

@@ -67,7 +67,7 @@ mock_domopipeline_config = {
                 "secretToken": "abcdefg",
                 "accessToken": "accessTpokem",
                 "apiHost": "api.domo.com",
-                "sandboxDomain": "https://domain.domo.com",
+                "instanceDomain": "https://domain.domo.com",
             }
         },
         "sourceConfig": {

openmetadata-docs/content/partials/v1.2/deployment/upgrade/upgrade-prerequisites.md

@@ -112,4 +112,6 @@ then there is no way to link a query to a service and the query will be removed.
 
 ### Service Connection Changes
 
+- Domo Database, Dashboard and Pipeline renamed the `sandboxDomain` in favor of `instanceDomain`.
+
 ### Other Changes

openmetadata-service/src/main/resources/json/data/testConnections/dashboard/domodashboard.json

@@ -9,6 +9,12 @@
         "errorMessage": "Failed to fetch dashboards, please validate the credentials or validate if user has access to fetch dashboards",
         "shortCircuit": true,
         "mandatory": true
+      },
+      {
+        "name": "GetCharts",
+        "description": "We can get the chart information using the Access Token and the Instance Domain.",
+        "errorMessage": "Failed to fetch chart information. Please make sure that the Access Token used has enough permissions to get chart details.",
+        "mandatory": false
       }
     ]
   }

openmetadata-service/src/test/java/org/openmetadata/service/resources/query/QueryResourceTest.java

@@ -119,7 +119,11 @@ public class QueryResourceTest extends EntityResourceTest<Query, CreateQuery> {
 
   @Test
   void post_without_query_400() {
-    CreateQuery create = new CreateQuery().withDuration(0.0).withQueryDate(1673857635064L);
+    CreateQuery create =
+        new CreateQuery()
+            .withDuration(0.0)
+            .withQueryDate(1673857635064L)
+            .withService(SNOWFLAKE_REFERENCE.getFullyQualifiedName());
     assertResponse(
         () -> createEntity(create, ADMIN_AUTH_HEADERS), Response.Status.BAD_REQUEST, "[query must not be null]");
   }

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/dashboard/domoDashboardConnection.json

@@ -44,9 +44,9 @@
         "type": "string",
         "format": "string"
       },
-      "sandboxDomain": {
+      "instanceDomain": {
-        "title": "Sandbox Domain",
+        "title": "Instance Domain",
-        "description": "Connect to Sandbox Domain",
+        "description": "URL of your Domo instance, e.g., https://openmetadata.domo.com",
         "type": "string",
         "format": "uri"
       },
@@ -56,6 +56,6 @@
       }
     },
     "additionalProperties": false,
-    "required": ["clientId", "secretToken", "sandboxDomain"]
+    "required": ["clientId", "secretToken", "instanceDomain"]
   }
   

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/domoDatabaseConnection.json

@@ -44,9 +44,9 @@
         "type": "string",
         "format": "string"
       },
-      "sandboxDomain": {
+      "instanceDomain": {
-        "title": "Sandbox Domain",
+        "title": "Instance Domain",
-        "description": "Connect to Sandbox Domain",
+        "description": "URL of your Domo instance, e.g., https://openmetadata.domo.com",
         "type": "string",
         "format": "uri"
       },
@@ -61,6 +61,6 @@
       }
     },
     "additionalProperties": false,
-    "required": ["clientId", "secretToken", "sandboxDomain"]
+    "required": ["clientId", "secretToken", "instanceDomain"]
   }
   

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/pipeline/domoPipelineConnection.json

@@ -44,9 +44,9 @@
         "type": "string",
         "format": "string"
       },
-      "sandboxDomain": {
+      "instanceDomain": {
-        "title": "Sandbox Domain",
+        "title": "Instance Domain",
-        "description": "Connect to Sandbox Domain",
+        "description": "URL of your Domo instance, e.g., https://openmetadata.domo.com",
         "type": "string",
         "format": "uri"
       },
@@ -56,6 +56,6 @@
       }
     },
     "additionalProperties": false,
-    "required": ["clientId","secretToken","sandboxDomain"]
+    "required": ["clientId","secretToken","instanceDomain"]
   }
   

openmetadata-spec/src/main/resources/json/schema/metadataIngestion/databaseServiceProfilerPipeline.json

@@ -43,7 +43,7 @@
       "description": "Optional configuration to automatically tag columns that might contain sensitive information",
       "type": "boolean",
       "default": false,
-      "title": "Process PII Sensitive"
+      "title": "Auto Tag PII"
     },
     "confidence": {
       "description": "Set the Confidence value for which you want the column to be marked",

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Dashboard/DomoDashboard.md

@@ -14,6 +14,8 @@ $$section
 ### Client ID $(id="clientId")
 
 Client ID for Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#how-to-find-clientid).
+
+This needs to be informed together with the `Secret Token` and is used to extract metadata from Domo's official API.
 $$
 
 $$section
@@ -26,6 +28,8 @@ $$section
 ### Access Token $(id="accessToken")
 
 Access token to connect to Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#where-to-find-accesstoken).
+
+This is required to automate metadata extraction directly from the instance for endpoints not supported by the API, such as Cards or Pipeline Runs.
 $$
 
 $$section
@@ -35,7 +39,7 @@ API Host to connect your Domo instance. By default: `api.domo.com`.
 $$
 
 $$section
-### Sandbox Domain $(id="sandboxDomain")
+### Instance Domain $(id="instanceDomain")
 
-Connection to your Sandbox Domain. For example `https://<your>.domo.com`.
+URL to connect to your Domo instance UI. For example `https://<your>.domo.com`.
 $$

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Database/DomoDatabase.md

@@ -14,6 +14,8 @@ $$section
 ### Client ID $(id="clientId")
 
 Client ID for Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#how-to-find-clientid).
+
+This needs to be informed together with the `Secret Token` and is used to extract metadata from Domo's official API.
 $$
 
 $$section
@@ -26,6 +28,8 @@ $$section
 ### Access Token $(id="accessToken")
 
 Access token to connect to Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#where-to-find-accesstoken).
+
+This is required to automate metadata extraction directly from the instance for endpoints not supported by the API, such as Cards or Pipeline Runs.
 $$
 
 $$section
@@ -35,9 +39,9 @@ API Host to connect your Domo instance. By default: `api.domo.com`.
 $$
 
 $$section
-### Sandbox Domain $(id="sandboxDomain")
+### Instance Domain $(id="instanceDomain")
 
-Connection to your Sandbox Domain. For example `https://<your>.domo.com`.
+URL to connect to your Domo instance UI. For example `https://<your>.domo.com`.
 $$
 
 $$section

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Pipeline/DomoPipeline.md

@@ -14,6 +14,8 @@ $$section
 ### Client ID $(id="clientId")
 
 Client ID for Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#how-to-find-clientid).
+
+This needs to be informed together with the `Secret Token` and is used to extract metadata from Domo's official API.
 $$
 
 $$section
@@ -26,6 +28,8 @@ $$section
 ### Access Token $(id="accessToken")
 
 Access token to connect to Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#where-to-find-accesstoken).
+
+This is required to automate metadata extraction directly from the instance for endpoints not supported by the API, such as Cards or Pipeline Runs.
 $$
 
 $$section
@@ -35,7 +39,7 @@ API Host to connect your Domo instance. By default: `api.domo.com`.
 $$
 
 $$section
-### Sandbox Domain $(id="sandboxDomain")
+### Instance Domain $(id="instanceDomain")
 
-Connection to your Sandbox Domain. For example `https://<your>.domo.com`.
+URL to connect to your Domo instance UI. For example `https://<your>.domo.com`.
 $$