yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2025-12-24 14:08:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ingestion: Metabase Unit Tests (#11080)

Browse Source
This commit is contained in:
Mayur Singal 2023-04-18 09:08:17 +05:30 committed by GitHub
parent e7013b481a
commit 857ddeab1e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 377 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py
View File

@ -12,6 +12,7 @@
REST Auth & Client for Metabase
"""
import json
import traceback
from typing import List, Optional
import requests
@ -81,11 +82,14 @@ class MetabaseClient:
"""
Get List of all dashboards
"""
resp_dashboards = self.client.get("/dashboard")
if resp_dashboards:
dashboard_list = MetabaseDashboardList(dashboards=resp_dashboards)
return dashboard_list.dashboards
logger.warning(f"Failed to fetch the dashboards: {resp_dashboards.text}")
try:
resp_dashboards = self.client.get("/dashboard")
if resp_dashboards:
dashboard_list = MetabaseDashboardList(dashboards=resp_dashboards)
return dashboard_list.dashboards
except Exception:
logger.debug(traceback.format_exc())
logger.warning("Failed to fetch the dashboard list")
return []
def get_dashboard_details(
@ -94,28 +98,43 @@ class MetabaseClient:
"""
Get Dashboard Details
"""
resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}")
if resp_dashboard:
return MetabaseDashboardDetails(**resp_dashboard)
logger.warning(f"Failed to fetch the dashboard: {resp_dashboard.text}")
if not dashboard_id:
return None # don't call api if dashboard_id is None
try:
resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}")
if resp_dashboard:
return MetabaseDashboardDetails(**resp_dashboard)
except Exception:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}")
return None
def get_database(self, database_id: str) -> Optional[MetabaseDatabase]:
"""
Get Database using database ID
"""
resp_database = self.client.get(f"/database/{database_id}")
if resp_database:
return MetabaseDatabase(**resp_database)
logger.warning(f"Failed to fetch the database: {resp_database.text}")
if not database_id:
return None # don't call api if database_id is None
try:
resp_database = self.client.get(f"/database/{database_id}")
if resp_database:
return MetabaseDatabase(**resp_database)
except Exception:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch the database with id: {database_id}")
return None
def get_table(self, table_id: str) -> Optional[MetabaseTable]:
"""
Get Table using table ID
"""
resp_table = self.client.get(f"/table/{table_id}")
if resp_table:
return MetabaseTable(**resp_table)
logger.warning(f"Failed to fetch the table: {resp_table.text}")
if not table_id:
return None # don't call api if table_id is None
try:
resp_table = self.client.get(f"/table/{table_id}")
if resp_table:
return MetabaseTable(**resp_table)
except Exception:
logger.debug(traceback.format_exc())
logger.warning(f"Failed to fetch the table with id: {table_id}")
return None

57
ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py
View File

@ -92,28 +92,34 @@ class MetabaseSource(DashboardServiceSource):
"""
Method to Get Dashboard Entity
"""
dashboard_url = (
f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-"
f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
)
dashboard_request = CreateDashboardRequest(
name=dashboard_details.id,
dashboardUrl=dashboard_url,
displayName=dashboard_details.name,
description=dashboard_details.description,
charts=[
fqn.build(
self.metadata,
entity_type=Chart,
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
chart_name=chart.name.__root__,
)
for chart in self.context.charts
],
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
yield dashboard_request
self.register_record(dashboard_request=dashboard_request)
try:
dashboard_url = (
f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-"
f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}"
)
dashboard_request = CreateDashboardRequest(
name=dashboard_details.id,
dashboardUrl=dashboard_url,
displayName=dashboard_details.name,
description=dashboard_details.description,
charts=[
fqn.build(
self.metadata,
entity_type=Chart,
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
chart_name=chart.name.__root__,
)
for chart in self.context.charts
],
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
yield dashboard_request
self.register_record(dashboard_request=dashboard_request)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(
f"Error creating dashboard [{dashboard_details.name}]: {exc}"
)
def yield_dashboard_chart(
self, dashboard_details: MetabaseDashboardDetails
@ -152,7 +158,6 @@ class MetabaseSource(DashboardServiceSource):
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(f"Error creating chart [{chart}]: {exc}")
continue
def yield_dashboard_lineage_details(
self,
@ -179,8 +184,6 @@ class MetabaseSource(DashboardServiceSource):
):
continue
if chart_details.dataset_query.type == "native":
if not chart_details.database_id:
continue
yield from self._yield_lineage_from_query(
chart_details=chart_details,
db_service_name=db_service_name,
@ -215,10 +218,10 @@ class MetabaseSource(DashboardServiceSource):
):
query = chart_details.dataset_query.native.query
if database is None or query is None:
if query is None:
return
database_name = database.details.db if database.details else None
database_name = database.details.db if database and database.details else None
lineage_parser = LineageParser(query)
for table in lineage_parser.source_tables:

8
ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py
View File

@ -11,7 +11,7 @@
"""
Metabase Models
"""
from typing import Any, List, Optional
from typing import List, Optional
from pydantic import BaseModel, Field
@ -30,12 +30,6 @@ class MetabaseDashboardList(BaseModel):
dashboards: Optional[List[MetabaseDashboard]]
class ResultMetadata(BaseModel):
display_name: Optional[str]
field_ref: Any
name: Optional[str]
class Native(BaseModel):
query: Optional[str]

11
ingestion/tests/unit/topology/dashboard/test_domodashboard.py
View File

@ -1,3 +1,14 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Test Domo Dashboard using the topology
"""

288
ingestion/tests/unit/topology/dashboard/test_metabase.py Normal file
View File

@ -0,0 +1,288 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Test Domo Dashboard using the topology
"""
from copy import deepcopy
from types import SimpleNamespace
from unittest import TestCase
from unittest.mock import patch
from metadata.generated.schema.api.data.createChart import CreateChartRequest
from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest
from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest
from metadata.generated.schema.entity.data.dashboard import (
Dashboard as LineageDashboard,
)
from metadata.generated.schema.entity.data.table import Table
from metadata.generated.schema.entity.services.dashboardService import (
DashboardConnection,
DashboardService,
DashboardServiceType,
)
from metadata.generated.schema.metadataIngestion.workflow import (
OpenMetadataWorkflowConfig,
)
from metadata.generated.schema.type.basic import FullyQualifiedEntityName
from metadata.generated.schema.type.entityLineage import EntitiesEdge
from metadata.generated.schema.type.entityReference import EntityReference
from metadata.ingestion.ometa.ometa_api import OpenMetadata
from metadata.ingestion.source.dashboard.metabase import metadata as MetabaseMetadata
from metadata.ingestion.source.dashboard.metabase.metadata import MetabaseSource
from metadata.ingestion.source.dashboard.metabase.models import (
DatasetQuery,
MetabaseChart,
MetabaseDashboardDetails,
MetabaseTable,
Native,
OrderedCard,
)
from metadata.utils import fqn
MOCK_DASHBOARD_SERVICE = DashboardService(
id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb",
fullyQualifiedName=FullyQualifiedEntityName(__root__="mock_metabase"),
name="mock_metabase",
connection=DashboardConnection(),
serviceType=DashboardServiceType.Metabase,
)
EXAMPLE_DASHBOARD = LineageDashboard(
id="7b3766b1-7eb4-4ad4-b7c8-15a8b16edfdd",
name="lineage_dashboard",
service=EntityReference(
id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb", type="dashboardService"
),
)
EXAMPLE_TABLE = [
Table(
id="0bd6bd6f-7fea-4a98-98c7-3b37073629c7",
name="lineage_table",
columns=[],
)
]
mock_tableau_config = {
"source": {
"type": "metabase",
"serviceName": "mock_metabase",
"serviceConnection": {
"config": {
"type": "Metabase",
"username": "username",
"password": "abcdefg",
"hostPort": "http://metabase.com",
}
},
"sourceConfig": {
"config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}}
},
},
"sink": {"type": "metadata-rest", "config": {}},
"workflowConfig": {
"loggerLevel": "DEBUG",
"openMetadataServerConfig": {
"hostPort": "http://localhost:8585/api",
"authProvider": "openmetadata",
"securityConfig": {
"jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc"
"iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE"
"2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB"
"iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN"
"r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u"
"d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg"
},
},
},
}
MOCK_CHARTS = [
OrderedCard(
card=MetabaseChart(
description="Test Chart",
table_id=1,
database_id=1,
name="chart1",
id="1",
dataset_query=DatasetQuery(type="query"),
display="chart1",
)
),
OrderedCard(
card=MetabaseChart(
description="Test Chart",
table_id=1,
database_id=1,
name="chart2",
id="2",
dataset_query=DatasetQuery(
type="native", native=Native(query="select * from table")
),
display="chart2",
)
),
OrderedCard(card=MetabaseChart(name="chart3", id="3")),
]
EXPECTED_LINEAGE = AddLineageRequest(
edge=EntitiesEdge(
fromEntity=EntityReference(
id="0bd6bd6f-7fea-4a98-98c7-3b37073629c7",
type="table",
),
toEntity=EntityReference(
id="7b3766b1-7eb4-4ad4-b7c8-15a8b16edfdd",
type="dashboard",
),
)
)
MOCK_DASHBOARD_DETAILS = MetabaseDashboardDetails(
description="SAMPLE DESCRIPTION", name="test_db", id="1", ordered_cards=MOCK_CHARTS
)
EXPECTED_DASHBOARD = [
CreateDashboardRequest(
name="1",
displayName="test_db",
description="SAMPLE DESCRIPTION",
dashboardUrl="http://metabase.com/dashboard/1-test-db",
charts=[],
service=FullyQualifiedEntityName(__root__="mock_metabase"),
)
]
EXPECTED_CHARTS = [
CreateChartRequest(
name="1",
displayName="chart1",
description="Test Chart",
chartType="Other",
chartUrl="http://metabase.com/question/1-chart1",
tags=None,
owner=None,
service=FullyQualifiedEntityName(__root__="mock_metabase"),
),
CreateChartRequest(
name="2",
displayName="chart2",
description="Test Chart",
chartType="Other",
chartUrl="http://metabase.com/question/2-chart2",
tags=None,
owner=None,
service=FullyQualifiedEntityName(__root__="mock_metabase"),
),
CreateChartRequest(
name="3",
displayName="chart3",
description=None,
chartType="Other",
chartUrl="http://metabase.com/question/3-chart3",
tags=None,
owner=None,
service=FullyQualifiedEntityName(__root__="mock_metabase"),
),
]
class MetabaseUnitTest(TestCase):
"""
Implements the necessary methods to extract
Domo Dashboard Unit Test
"""
@patch(
"metadata.ingestion.source.dashboard.dashboard_service.DashboardServiceSource.test_connection"
)
@patch("metadata.ingestion.source.dashboard.metabase.connection.get_connection")
def __init__(self, methodName, get_connection, test_connection) -> None:
super().__init__(methodName)
get_connection.return_value = False
test_connection.return_value = False
self.config = OpenMetadataWorkflowConfig.parse_obj(mock_tableau_config)
self.metabase = MetabaseSource.create(
mock_tableau_config["source"],
self.config.workflowConfig.openMetadataServerConfig,
)
self.metabase.client = SimpleNamespace()
self.metabase.context.__dict__["dashboard_service"] = MOCK_DASHBOARD_SERVICE
def test_dashboard_name(self):
assert (
self.metabase.get_dashboard_name(MOCK_DASHBOARD_DETAILS)
== MOCK_DASHBOARD_DETAILS.name
)
def test_yield_chart(self):
"""
Function for testing charts
"""
chart_list = []
results = self.metabase.yield_dashboard_chart(MOCK_DASHBOARD_DETAILS)
for result in results:
if isinstance(result, CreateChartRequest):
chart_list.append(result)
for exptected, original in zip(EXPECTED_CHARTS, chart_list):
self.assertEqual(exptected, original)
def test_yield_dashboard(self):
"""
Function for testing charts
"""
results = list(self.metabase.yield_dashboard(MOCK_DASHBOARD_DETAILS))
self.assertEqual(EXPECTED_DASHBOARD, list(results))
@patch.object(fqn, "build", return_value=None)
@patch.object(OpenMetadata, "get_by_name", return_value=EXAMPLE_DASHBOARD)
@patch.object(MetabaseMetadata, "search_table_entities", return_value=EXAMPLE_TABLE)
def test_yield_lineage(self, *_):
"""
Function to test out lineage
"""
self.metabase.client.get_database = lambda *_: None
self.metabase.client.get_table = lambda *_: MetabaseTable(
schema="test_schema", display_name="test_table"
)
# if no db service name then no lineage generated
result = self.metabase.yield_dashboard_lineage_details(
dashboard_details=MOCK_DASHBOARD_DETAILS, db_service_name=None
)
self.assertEqual(list(result), [])
# test out _yield_lineage_from_api
mock_dashboard = deepcopy(MOCK_DASHBOARD_DETAILS)
mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[0]]
result = self.metabase.yield_dashboard_lineage_details(
dashboard_details=mock_dashboard, db_service_name="db.service.name"
)
self.assertEqual(next(result), EXPECTED_LINEAGE)
# test out _yield_lineage_from_query
mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[1]]
result = self.metabase.yield_dashboard_lineage_details(
dashboard_details=mock_dashboard, db_service_name="db.service.name"
)
self.assertEqual(next(result), EXPECTED_LINEAGE)
# test out if no query type
mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[2]]
result = self.metabase.yield_dashboard_lineage_details(
dashboard_details=mock_dashboard, db_service_name="db.service.name"
)
self.assertEqual(list(result), [])

11
ingestion/tests/unit/topology/dashboard/test_quicksight.py
View File

@ -1,3 +1,14 @@
# Copyright 2021 Collate
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Test QuickSight using the topology
"""