Fix #13237 - Rename to instanceDomain and test DomoDashboard charts (#13247)

* Rename sandboxDomain to instanceDomain

* Test Get Charts in DomoDashboard

* Fix schemas

* Fix test

* Fix test

* Rename to Auto Tag PII

* Fix query test

* Fix query test

* Fix query test
This commit is contained in:
Pere Miquel Brull 2023-09-19 14:14:04 +02:00 committed by GitHub
parent c2ed4f422f
commit 18a4513ccc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 162 additions and 57 deletions

View File

@ -85,5 +85,30 @@ SET json = JSON_INSERT(
)
WHERE JSON_EXTRACT(json, '$.pipelineType') = 'metadata';
-- Rename sandboxDomain for instanceDomain
UPDATE dbservice_entity
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.connection.config.sandboxDomain'),
'$.connection.config.instanceDomain',
JSON_EXTRACT(json, '$.connection.config.sandboxDomain')
)
WHERE serviceType = 'DomoDatabase';
UPDATE dashboard_service_entity
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.connection.config.sandboxDomain'),
'$.connection.config.instanceDomain',
JSON_EXTRACT(json, '$.connection.config.sandboxDomain')
)
WHERE serviceType = 'DomoDashboard';
UPDATE pipeline_service_entity
SET json = JSON_INSERT(
JSON_REMOVE(json, '$.connection.config.sandboxDomain'),
'$.connection.config.instanceDomain',
JSON_EXTRACT(json, '$.connection.config.sandboxDomain')
)
WHERE serviceType = 'DomoPipeline';
-- Query Entity supports service, which requires FQN for name
ALTER TABLE query_entity CHANGE COLUMN nameHash fqnHash VARCHAR(256);

View File

@ -102,5 +102,33 @@ SET json = jsonb_set(
)
WHERE json #>> '{pipelineType}' = 'metadata';
-- Rename sandboxDomain for instanceDomain
UPDATE dbservice_entity
SET json = jsonb_set(
json::jsonb #- '{connection,config,sandboxDomain}',
'{connection,config,instanceDomain}',
(json #> '{connection,config,sandboxDomain}')::jsonb,
true
)
WHERE serviceType = 'DomoDatabase';
UPDATE dashboard_service_entity
SET json = jsonb_set(
json::jsonb #- '{connection,config,sandboxDomain}',
'{connection,config,instanceDomain}',
(json #> '{connection,config,sandboxDomain}')::jsonb,
true
)
WHERE serviceType = 'DomoDashboard';
UPDATE pipeline_service_entity
SET json = jsonb_set(
json::jsonb #- '{connection,config,sandboxDomain}',
'{connection,config,instanceDomain}',
(json #> '{connection,config,sandboxDomain}')::jsonb,
true
)
WHERE serviceType = 'DomoPipeline';
-- Query Entity supports service, which requires FQN for name
ALTER TABLE query_entity RENAME COLUMN nameHash TO fqnHash;

View File

@ -14,9 +14,11 @@ DomoClient source to extract data from DOMO
"""
import traceback
from dataclasses import dataclass
from typing import List, Optional, Union
from pydantic import BaseModel, Extra
from pydomo import Domo
from metadata.generated.schema.entity.services.connections.dashboard.domoDashboardConnection import (
DomoDashboardConnection,
@ -101,14 +103,14 @@ class DomoClient:
],
):
self.config = config
self.config.sandboxDomain = (
self.config.sandboxDomain[:-1]
if self.config.sandboxDomain.endswith("/")
else self.config.sandboxDomain
self.config.instanceDomain = (
self.config.instanceDomain[:-1]
if self.config.instanceDomain.endswith("/")
else self.config.instanceDomain
)
HEADERS.update({"X-DOMO-Developer-Token": self.config.accessToken})
client_config: ClientConfig = ClientConfig(
base_url=self.config.sandboxDomain,
base_url=self.config.instanceDomain,
api_version="api/",
auth_header="Authorization",
auth_token=lambda: ("no_token", 0),
@ -170,3 +172,29 @@ class DomoClient:
)
logger.debug(traceback.format_exc())
return []
def test_list_cards(self) -> None:
"""
Test function to list the cards. Since we are not passing any URNS from the dashboard
we expect an empty result. However, the call should not fail with any 401 error.
This helps us validate that the provided Access Token is correct for Domo Dashboard.
"""
try:
self.client._request( # pylint: disable=protected-access
method="GET", path="content/v1/cards", headers=HEADERS
)
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Error listing cards due to [{exc}]")
raise exc
@dataclass
class OMPyDomoClient:
"""
domo_client: official pydomo client https://github.com/domoinc/domo-python-sdk
client: custom requests on the instance domain
"""
domo: Domo
custom: DomoClient

View File

@ -8,7 +8,7 @@ source:
secretToken: secret-token
accessToken: access-token
apiHost: api.domo.com
sandboxDomain: https://api_domo.domo.com
instanceDomain: https://api_domo.domo.com
sourceConfig:
config:
type: DashboardMetadata

View File

@ -17,6 +17,7 @@ from typing import Optional
from pydomo import Domo
from metadata.clients.domo_client import DomoClient, OMPyDomoClient
from metadata.generated.schema.entity.automations.workflow import (
Workflow as AutomationWorkflow,
)
@ -30,7 +31,7 @@ from metadata.ingestion.connections.test_connections import (
from metadata.ingestion.ometa.ometa_api import OpenMetadata
def get_connection(connection: DomoDashboardConnection) -> Domo:
def get_connection(connection: DomoDashboardConnection) -> OMPyDomoClient:
"""
Create connection
"""
@ -40,7 +41,12 @@ def get_connection(connection: DomoDashboardConnection) -> Domo:
connection.secretToken.get_secret_value(),
api_host=connection.apiHost,
)
return domo
client = DomoClient(connection)
return OMPyDomoClient(
domo=domo,
custom=client,
)
except Exception as exc:
msg = f"Unknown error connecting with {connection}: {exc}."
raise SourceConnectionException(msg)
@ -48,7 +54,7 @@ def get_connection(connection: DomoDashboardConnection) -> Domo:
def test_connection(
metadata: OpenMetadata,
domo: Domo,
client: OMPyDomoClient,
service_connection: DomoDashboardConnection,
automation_workflow: Optional[AutomationWorkflow] = None,
) -> None:
@ -57,11 +63,14 @@ def test_connection(
of a metadata workflow or during an Automation Workflow
"""
def custom_executor():
result = domo.page_list()
def custom_test_page_list():
result = client.domo.page_list()
return list(result)
test_fn = {"GetDashboards": custom_executor}
test_fn = {
"GetDashboards": custom_test_page_list,
"GetCharts": client.custom.test_list_cards,
}
test_connection_steps(
metadata=metadata,

View File

@ -19,7 +19,6 @@ from pydantic import ValidationError
from metadata.clients.domo_client import (
DomoChartDetails,
DomoClient,
DomoDashboardDetails,
DomoOwner,
)
@ -57,10 +56,6 @@ class DomodashboardSource(DashboardServiceSource):
config: WorkflowSource
metadata_config: OpenMetadataConnection
def __init__(self, config: WorkflowSource, metadata_config: OpenMetadataConnection):
super().__init__(config, metadata_config)
self.domo_client = DomoClient(self.service_connection)
@classmethod
def create(cls, config_dict, metadata_config: OpenMetadataConnection):
config = WorkflowSource.parse_obj(config_dict)
@ -72,7 +67,7 @@ class DomodashboardSource(DashboardServiceSource):
return cls(config, metadata_config)
def get_dashboards_list(self) -> Optional[List[DomoDashboardDetails]]:
dashboards = self.client.page_list()
dashboards = self.client.domo.page_list()
dashboard_list = []
for dashboard in dashboards:
dashboard_detail = self.get_page_details(page_id=dashboard["id"])
@ -99,7 +94,7 @@ class DomodashboardSource(DashboardServiceSource):
) -> Optional[EntityReference]:
for owner in dashboard_details.owners:
try:
owner_details = self.client.users_get(owner.id)
owner_details = self.client.domo.users_get(owner.id)
if owner_details.get("email"):
user = self.metadata.get_user_by_email(owner_details["email"])
if user:
@ -118,7 +113,7 @@ class DomodashboardSource(DashboardServiceSource):
) -> Iterable[Either[CreateDashboardRequest]]:
try:
dashboard_url = (
f"{self.service_connection.sandboxDomain}/page/{dashboard_details.id}"
f"{self.service_connection.instanceDomain}/page/{dashboard_details.id}"
)
dashboard_request = CreateDashboardRequest(
@ -175,7 +170,7 @@ class DomodashboardSource(DashboardServiceSource):
def get_page_details(self, page_id) -> Optional[DomoDashboardDetails]:
try:
pages = self.client.page_get(page_id)
pages = self.client.domo.page_get(page_id)
return DomoDashboardDetails(
name=pages["name"],
id=pages["id"],
@ -208,9 +203,9 @@ class DomodashboardSource(DashboardServiceSource):
for chart_id in chart_ids:
chart: Optional[DomoChartDetails] = None
try:
chart = self.domo_client.get_chart_details(page_id=chart_id)
chart = self.client.custom.get_chart_details(page_id=chart_id)
chart_url = (
f"{self.service_connection.sandboxDomain}/page/"
f"{self.service_connection.instanceDomain}/page/"
f"{dashboard_details.id}/kpis/details/{chart_id}"
)

View File

@ -16,7 +16,6 @@ Domo Database source to extract metadata
import traceback
from typing import Any, Iterable, List, Optional, Tuple
from metadata.clients.domo_client import DomoClient
from metadata.generated.schema.api.data.createDatabase import CreateDatabaseRequest
from metadata.generated.schema.api.data.createDatabaseSchema import (
CreateDatabaseSchemaRequest,
@ -81,7 +80,6 @@ class DomodatabaseSource(DatabaseServiceSource):
self.service_connection = self.config.serviceConnection.__root__.config
self.domo_client = get_connection(self.service_connection)
self.connection_obj = self.domo_client
self.client = DomoClient(self.service_connection)
self.test_connection()
@classmethod
@ -258,7 +256,7 @@ class DomodatabaseSource(DatabaseServiceSource):
Method to get the source url for domodatabase
"""
try:
return f"{clean_uri(self.service_connection.sandboxDomain)}/datasources/{table_name}/details/overview"
return f"{clean_uri(self.service_connection.instanceDomain)}/datasources/{table_name}/details/overview"
except Exception as exc:
logger.debug(traceback.format_exc())
logger.warning(f"Unable to get source url for {table_name}: {exc}")
@ -270,4 +268,4 @@ class DomodatabaseSource(DatabaseServiceSource):
return table
def close(self) -> None:
self.client.client.close()
"""Nothing to close"""

View File

@ -171,7 +171,7 @@ class DomopipelineSource(PipelineServiceSource):
) -> Optional[str]:
try:
return (
f"{clean_uri(self.service_connection.sandboxDomain)}/datacenter/dataflows/"
f"{clean_uri(self.service_connection.instanceDomain)}/datacenter/dataflows/"
f"{pipeline_id}/details#history"
)
except Exception as exc:

View File

@ -62,7 +62,7 @@ mock_domopipeline_config = {
"secretToken": "abcdefg",
"accessToken": "accessTpokem",
"apiHost": "api.domo.com",
"sandboxDomain": "https://domain.domo.com",
"instanceDomain": "https://domain.domo.com",
}
},
"sourceConfig": {
@ -187,12 +187,10 @@ class DomoDashboardUnitTest(TestCase):
for _, (expected, original) in enumerate(zip(EXPECTED_CHARTS, chart_list)):
self.assertEqual(expected, original)
# Cover error responses
with patch.object(REST, "_request", return_value=mock_data[1]):
result = self.domodashboard.domo_client.get_chart_details(
MOCK_DASHBOARD.cardIds[0]
)
assert (
self.domodashboard.domo_client.get_chart_details(
self.domodashboard.client.custom.get_chart_details(
MOCK_DASHBOARD.cardIds[0]
)
is None
@ -200,7 +198,7 @@ class DomoDashboardUnitTest(TestCase):
with patch.object(REST, "_request", return_value=mock_data[2]):
assert (
self.domodashboard.domo_client.get_chart_details(
self.domodashboard.client.custom.get_chart_details(
MOCK_DASHBOARD.cardIds[0]
)
is None

View File

@ -84,7 +84,7 @@ mock_domodatabase_config = {
"secretToken": "abcdefg",
"accessToken": "accessTpokem",
"apiHost": "api.domo.com",
"sandboxDomain": "https://domain.domo.com",
"instanceDomain": "https://domain.domo.com",
}
},
"sourceConfig": {"config": {"type": "DatabaseMetadata"}},

View File

@ -67,7 +67,7 @@ mock_domopipeline_config = {
"secretToken": "abcdefg",
"accessToken": "accessTpokem",
"apiHost": "api.domo.com",
"sandboxDomain": "https://domain.domo.com",
"instanceDomain": "https://domain.domo.com",
}
},
"sourceConfig": {

View File

@ -112,4 +112,6 @@ then there is no way to link a query to a service and the query will be removed.
### Service Connection Changes
- Domo Database, Dashboard and Pipeline renamed the `sandboxDomain` in favor of `instanceDomain`.
### Other Changes

View File

@ -9,6 +9,12 @@
"errorMessage": "Failed to fetch dashboards, please validate the credentials or validate if user has access to fetch dashboards",
"shortCircuit": true,
"mandatory": true
},
{
"name": "GetCharts",
"description": "We can get the chart information using the Access Token and the Instance Domain.",
"errorMessage": "Failed to fetch chart information. Please make sure that the Access Token used has enough permissions to get chart details.",
"mandatory": false
}
]
}

View File

@ -119,7 +119,11 @@ public class QueryResourceTest extends EntityResourceTest<Query, CreateQuery> {
@Test
void post_without_query_400() {
CreateQuery create = new CreateQuery().withDuration(0.0).withQueryDate(1673857635064L);
CreateQuery create =
new CreateQuery()
.withDuration(0.0)
.withQueryDate(1673857635064L)
.withService(SNOWFLAKE_REFERENCE.getFullyQualifiedName());
assertResponse(
() -> createEntity(create, ADMIN_AUTH_HEADERS), Response.Status.BAD_REQUEST, "[query must not be null]");
}

View File

@ -44,9 +44,9 @@
"type": "string",
"format": "string"
},
"sandboxDomain": {
"title": "Sandbox Domain",
"description": "Connect to Sandbox Domain",
"instanceDomain": {
"title": "Instance Domain",
"description": "URL of your Domo instance, e.g., https://openmetadata.domo.com",
"type": "string",
"format": "uri"
},
@ -56,6 +56,6 @@
}
},
"additionalProperties": false,
"required": ["clientId", "secretToken", "sandboxDomain"]
"required": ["clientId", "secretToken", "instanceDomain"]
}

View File

@ -44,9 +44,9 @@
"type": "string",
"format": "string"
},
"sandboxDomain": {
"title": "Sandbox Domain",
"description": "Connect to Sandbox Domain",
"instanceDomain": {
"title": "Instance Domain",
"description": "URL of your Domo instance, e.g., https://openmetadata.domo.com",
"type": "string",
"format": "uri"
},
@ -61,6 +61,6 @@
}
},
"additionalProperties": false,
"required": ["clientId", "secretToken", "sandboxDomain"]
"required": ["clientId", "secretToken", "instanceDomain"]
}

View File

@ -44,9 +44,9 @@
"type": "string",
"format": "string"
},
"sandboxDomain": {
"title": "Sandbox Domain",
"description": "Connect to Sandbox Domain",
"instanceDomain": {
"title": "Instance Domain",
"description": "URL of your Domo instance, e.g., https://openmetadata.domo.com",
"type": "string",
"format": "uri"
},
@ -56,6 +56,6 @@
}
},
"additionalProperties": false,
"required": ["clientId","secretToken","sandboxDomain"]
"required": ["clientId","secretToken","instanceDomain"]
}

View File

@ -43,7 +43,7 @@
"description": "Optional configuration to automatically tag columns that might contain sensitive information",
"type": "boolean",
"default": false,
"title": "Process PII Sensitive"
"title": "Auto Tag PII"
},
"confidence": {
"description": "Set the Confidence value for which you want the column to be marked",

View File

@ -14,6 +14,8 @@ $$section
### Client ID $(id="clientId")
Client ID for Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#how-to-find-clientid).
This needs to be informed together with the `Secret Token` and is used to extract metadata from Domo's official API.
$$
$$section
@ -26,6 +28,8 @@ $$section
### Access Token $(id="accessToken")
Access token to connect to Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#where-to-find-accesstoken).
This is required to automate metadata extraction directly from the instance for endpoints not supported by the API, such as Cards or Pipeline Runs.
$$
$$section
@ -35,7 +39,7 @@ API Host to connect your Domo instance. By default: `api.domo.com`.
$$
$$section
### Sandbox Domain $(id="sandboxDomain")
### Instance Domain $(id="instanceDomain")
Connection to your Sandbox Domain. For example `https://<your>.domo.com`.
URL to connect to your Domo instance UI. For example `https://<your>.domo.com`.
$$

View File

@ -14,6 +14,8 @@ $$section
### Client ID $(id="clientId")
Client ID for Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#how-to-find-clientid).
This needs to be informed together with the `Secret Token` and is used to extract metadata from Domo's official API.
$$
$$section
@ -26,6 +28,8 @@ $$section
### Access Token $(id="accessToken")
Access token to connect to Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#where-to-find-accesstoken).
This is required to automate metadata extraction directly from the instance for endpoints not supported by the API, such as Cards or Pipeline Runs.
$$
$$section
@ -35,9 +39,9 @@ API Host to connect your Domo instance. By default: `api.domo.com`.
$$
$$section
### Sandbox Domain $(id="sandboxDomain")
### Instance Domain $(id="instanceDomain")
Connection to your Sandbox Domain. For example `https://<your>.domo.com`.
URL to connect to your Domo instance UI. For example `https://<your>.domo.com`.
$$
$$section

View File

@ -14,6 +14,8 @@ $$section
### Client ID $(id="clientId")
Client ID for Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#how-to-find-clientid).
This needs to be informed together with the `Secret Token` and is used to extract metadata from Domo's official API.
$$
$$section
@ -26,6 +28,8 @@ $$section
### Access Token $(id="accessToken")
Access token to connect to Domo. Further information can be found [here](https://docs.open-metadata.org/connectors/database/domo-database/troubleshoot#where-to-find-accesstoken).
This is required to automate metadata extraction directly from the instance for endpoints not supported by the API, such as Cards or Pipeline Runs.
$$
$$section
@ -35,7 +39,7 @@ API Host to connect your Domo instance. By default: `api.domo.com`.
$$
$$section
### Sandbox Domain $(id="sandboxDomain")
### Instance Domain $(id="instanceDomain")
Connection to your Sandbox Domain. For example `https://<your>.domo.com`.
URL to connect to your Domo instance UI. For example `https://<your>.domo.com`.
$$