diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py index 6cf5c73e2b6..b519a21162f 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/client.py @@ -12,6 +12,7 @@ REST Auth & Client for Metabase """ import json +import traceback from typing import List, Optional import requests @@ -81,11 +82,14 @@ class MetabaseClient: """ Get List of all dashboards """ - resp_dashboards = self.client.get("/dashboard") - if resp_dashboards: - dashboard_list = MetabaseDashboardList(dashboards=resp_dashboards) - return dashboard_list.dashboards - logger.warning(f"Failed to fetch the dashboards: {resp_dashboards.text}") + try: + resp_dashboards = self.client.get("/dashboard") + if resp_dashboards: + dashboard_list = MetabaseDashboardList(dashboards=resp_dashboards) + return dashboard_list.dashboards + except Exception: + logger.debug(traceback.format_exc()) + logger.warning("Failed to fetch the dashboard list") return [] def get_dashboard_details( @@ -94,28 +98,43 @@ class MetabaseClient: """ Get Dashboard Details """ - resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}") - if resp_dashboard: - return MetabaseDashboardDetails(**resp_dashboard) - logger.warning(f"Failed to fetch the dashboard: {resp_dashboard.text}") + if not dashboard_id: + return None # don't call api if dashboard_id is None + try: + resp_dashboard = self.client.get(f"/dashboard/{dashboard_id}") + if resp_dashboard: + return MetabaseDashboardDetails(**resp_dashboard) + except Exception: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to fetch the dashboard with id: {dashboard_id}") return None def get_database(self, database_id: str) -> Optional[MetabaseDatabase]: """ Get Database using database ID """ - resp_database = self.client.get(f"/database/{database_id}") - if resp_database: - return MetabaseDatabase(**resp_database) - logger.warning(f"Failed to fetch the database: {resp_database.text}") + if not database_id: + return None # don't call api if database_id is None + try: + resp_database = self.client.get(f"/database/{database_id}") + if resp_database: + return MetabaseDatabase(**resp_database) + except Exception: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to fetch the database with id: {database_id}") return None def get_table(self, table_id: str) -> Optional[MetabaseTable]: """ Get Table using table ID """ - resp_table = self.client.get(f"/table/{table_id}") - if resp_table: - return MetabaseTable(**resp_table) - logger.warning(f"Failed to fetch the table: {resp_table.text}") + if not table_id: + return None # don't call api if table_id is None + try: + resp_table = self.client.get(f"/table/{table_id}") + if resp_table: + return MetabaseTable(**resp_table) + except Exception: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to fetch the table with id: {table_id}") return None diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py index 436bd4f5c83..2d319c26718 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/metadata.py @@ -92,28 +92,34 @@ class MetabaseSource(DashboardServiceSource): """ Method to Get Dashboard Entity """ - dashboard_url = ( - f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-" - f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}" - ) - dashboard_request = CreateDashboardRequest( - name=dashboard_details.id, - dashboardUrl=dashboard_url, - displayName=dashboard_details.name, - description=dashboard_details.description, - charts=[ - fqn.build( - self.metadata, - entity_type=Chart, - service_name=self.context.dashboard_service.fullyQualifiedName.__root__, - chart_name=chart.name.__root__, - ) - for chart in self.context.charts - ], - service=self.context.dashboard_service.fullyQualifiedName.__root__, - ) - yield dashboard_request - self.register_record(dashboard_request=dashboard_request) + try: + dashboard_url = ( + f"{clean_uri(self.service_connection.hostPort)}/dashboard/{dashboard_details.id}-" + f"{replace_special_with(raw=dashboard_details.name.lower(), replacement='-')}" + ) + dashboard_request = CreateDashboardRequest( + name=dashboard_details.id, + dashboardUrl=dashboard_url, + displayName=dashboard_details.name, + description=dashboard_details.description, + charts=[ + fqn.build( + self.metadata, + entity_type=Chart, + service_name=self.context.dashboard_service.fullyQualifiedName.__root__, + chart_name=chart.name.__root__, + ) + for chart in self.context.charts + ], + service=self.context.dashboard_service.fullyQualifiedName.__root__, + ) + yield dashboard_request + self.register_record(dashboard_request=dashboard_request) + except Exception as exc: # pylint: disable=broad-except + logger.debug(traceback.format_exc()) + logger.warning( + f"Error creating dashboard [{dashboard_details.name}]: {exc}" + ) def yield_dashboard_chart( self, dashboard_details: MetabaseDashboardDetails @@ -152,7 +158,6 @@ class MetabaseSource(DashboardServiceSource): except Exception as exc: # pylint: disable=broad-except logger.debug(traceback.format_exc()) logger.warning(f"Error creating chart [{chart}]: {exc}") - continue def yield_dashboard_lineage_details( self, @@ -179,8 +184,6 @@ class MetabaseSource(DashboardServiceSource): ): continue if chart_details.dataset_query.type == "native": - if not chart_details.database_id: - continue yield from self._yield_lineage_from_query( chart_details=chart_details, db_service_name=db_service_name, @@ -215,10 +218,10 @@ class MetabaseSource(DashboardServiceSource): ): query = chart_details.dataset_query.native.query - if database is None or query is None: + if query is None: return - database_name = database.details.db if database.details else None + database_name = database.details.db if database and database.details else None lineage_parser = LineageParser(query) for table in lineage_parser.source_tables: diff --git a/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py b/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py index ccd7da9631d..19b26f05a4d 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/metabase/models.py @@ -11,7 +11,7 @@ """ Metabase Models """ -from typing import Any, List, Optional +from typing import List, Optional from pydantic import BaseModel, Field @@ -30,12 +30,6 @@ class MetabaseDashboardList(BaseModel): dashboards: Optional[List[MetabaseDashboard]] -class ResultMetadata(BaseModel): - display_name: Optional[str] - field_ref: Any - name: Optional[str] - - class Native(BaseModel): query: Optional[str] diff --git a/ingestion/tests/unit/topology/dashboard/test_domodashboard.py b/ingestion/tests/unit/topology/dashboard/test_domodashboard.py index 51cf4c96b69..6d93491a333 100644 --- a/ingestion/tests/unit/topology/dashboard/test_domodashboard.py +++ b/ingestion/tests/unit/topology/dashboard/test_domodashboard.py @@ -1,3 +1,14 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Test Domo Dashboard using the topology """ diff --git a/ingestion/tests/unit/topology/dashboard/test_metabase.py b/ingestion/tests/unit/topology/dashboard/test_metabase.py new file mode 100644 index 00000000000..dfc6bb0ef25 --- /dev/null +++ b/ingestion/tests/unit/topology/dashboard/test_metabase.py @@ -0,0 +1,288 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Test Domo Dashboard using the topology +""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest import TestCase +from unittest.mock import patch + +from metadata.generated.schema.api.data.createChart import CreateChartRequest +from metadata.generated.schema.api.data.createDashboard import CreateDashboardRequest +from metadata.generated.schema.api.lineage.addLineage import AddLineageRequest +from metadata.generated.schema.entity.data.dashboard import ( + Dashboard as LineageDashboard, +) +from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.dashboardService import ( + DashboardConnection, + DashboardService, + DashboardServiceType, +) +from metadata.generated.schema.metadataIngestion.workflow import ( + OpenMetadataWorkflowConfig, +) +from metadata.generated.schema.type.basic import FullyQualifiedEntityName +from metadata.generated.schema.type.entityLineage import EntitiesEdge +from metadata.generated.schema.type.entityReference import EntityReference +from metadata.ingestion.ometa.ometa_api import OpenMetadata +from metadata.ingestion.source.dashboard.metabase import metadata as MetabaseMetadata +from metadata.ingestion.source.dashboard.metabase.metadata import MetabaseSource +from metadata.ingestion.source.dashboard.metabase.models import ( + DatasetQuery, + MetabaseChart, + MetabaseDashboardDetails, + MetabaseTable, + Native, + OrderedCard, +) +from metadata.utils import fqn + +MOCK_DASHBOARD_SERVICE = DashboardService( + id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb", + fullyQualifiedName=FullyQualifiedEntityName(__root__="mock_metabase"), + name="mock_metabase", + connection=DashboardConnection(), + serviceType=DashboardServiceType.Metabase, +) + +EXAMPLE_DASHBOARD = LineageDashboard( + id="7b3766b1-7eb4-4ad4-b7c8-15a8b16edfdd", + name="lineage_dashboard", + service=EntityReference( + id="c3eb265f-5445-4ad3-ba5e-797d3a3071bb", type="dashboardService" + ), +) + +EXAMPLE_TABLE = [ + Table( + id="0bd6bd6f-7fea-4a98-98c7-3b37073629c7", + name="lineage_table", + columns=[], + ) +] +mock_tableau_config = { + "source": { + "type": "metabase", + "serviceName": "mock_metabase", + "serviceConnection": { + "config": { + "type": "Metabase", + "username": "username", + "password": "abcdefg", + "hostPort": "http://metabase.com", + } + }, + "sourceConfig": { + "config": {"dashboardFilterPattern": {}, "chartFilterPattern": {}} + }, + }, + "sink": {"type": "metadata-rest", "config": {}}, + "workflowConfig": { + "loggerLevel": "DEBUG", + "openMetadataServerConfig": { + "hostPort": "http://localhost:8585/api", + "authProvider": "openmetadata", + "securityConfig": { + "jwtToken": "eyJraWQiOiJHYjM4OWEtOWY3Ni1nZGpzLWE5MmotMDI0MmJrOTQzNTYiLCJ0eXAiOiJKV1QiLCJhbGc" + "iOiJSUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsImlzQm90IjpmYWxzZSwiaXNzIjoib3Blbi1tZXRhZGF0YS5vcmciLCJpYXQiOjE" + "2NjM5Mzg0NjIsImVtYWlsIjoiYWRtaW5Ab3Blbm1ldGFkYXRhLm9yZyJ9.tS8um_5DKu7HgzGBzS1VTA5uUjKWOCU0B_j08WXB" + "iEC0mr0zNREkqVfwFDD-d24HlNEbrqioLsBuFRiwIWKc1m_ZlVQbG7P36RUxhuv2vbSp80FKyNM-Tj93FDzq91jsyNmsQhyNv_fN" + "r3TXfzzSPjHt8Go0FMMP66weoKMgW2PbXlhVKwEuXUHyakLLzewm9UMeQaEiRzhiTMU3UkLXcKbYEJJvfNFcLwSl9W8JCO_l0Yj3u" + "d-qt_nQYEZwqW6u5nfdQllN133iikV4fM5QZsMCnm8Rq1mvLR0y9bmJiD7fwM1tmJ791TUWqmKaTnP49U493VanKpUAfzIiOiIbhg" + }, + }, + }, +} + + +MOCK_CHARTS = [ + OrderedCard( + card=MetabaseChart( + description="Test Chart", + table_id=1, + database_id=1, + name="chart1", + id="1", + dataset_query=DatasetQuery(type="query"), + display="chart1", + ) + ), + OrderedCard( + card=MetabaseChart( + description="Test Chart", + table_id=1, + database_id=1, + name="chart2", + id="2", + dataset_query=DatasetQuery( + type="native", native=Native(query="select * from table") + ), + display="chart2", + ) + ), + OrderedCard(card=MetabaseChart(name="chart3", id="3")), +] + + +EXPECTED_LINEAGE = AddLineageRequest( + edge=EntitiesEdge( + fromEntity=EntityReference( + id="0bd6bd6f-7fea-4a98-98c7-3b37073629c7", + type="table", + ), + toEntity=EntityReference( + id="7b3766b1-7eb4-4ad4-b7c8-15a8b16edfdd", + type="dashboard", + ), + ) +) + +MOCK_DASHBOARD_DETAILS = MetabaseDashboardDetails( + description="SAMPLE DESCRIPTION", name="test_db", id="1", ordered_cards=MOCK_CHARTS +) + + +EXPECTED_DASHBOARD = [ + CreateDashboardRequest( + name="1", + displayName="test_db", + description="SAMPLE DESCRIPTION", + dashboardUrl="http://metabase.com/dashboard/1-test-db", + charts=[], + service=FullyQualifiedEntityName(__root__="mock_metabase"), + ) +] + +EXPECTED_CHARTS = [ + CreateChartRequest( + name="1", + displayName="chart1", + description="Test Chart", + chartType="Other", + chartUrl="http://metabase.com/question/1-chart1", + tags=None, + owner=None, + service=FullyQualifiedEntityName(__root__="mock_metabase"), + ), + CreateChartRequest( + name="2", + displayName="chart2", + description="Test Chart", + chartType="Other", + chartUrl="http://metabase.com/question/2-chart2", + tags=None, + owner=None, + service=FullyQualifiedEntityName(__root__="mock_metabase"), + ), + CreateChartRequest( + name="3", + displayName="chart3", + description=None, + chartType="Other", + chartUrl="http://metabase.com/question/3-chart3", + tags=None, + owner=None, + service=FullyQualifiedEntityName(__root__="mock_metabase"), + ), +] + + +class MetabaseUnitTest(TestCase): + """ + Implements the necessary methods to extract + Domo Dashboard Unit Test + """ + + @patch( + "metadata.ingestion.source.dashboard.dashboard_service.DashboardServiceSource.test_connection" + ) + @patch("metadata.ingestion.source.dashboard.metabase.connection.get_connection") + def __init__(self, methodName, get_connection, test_connection) -> None: + super().__init__(methodName) + get_connection.return_value = False + test_connection.return_value = False + self.config = OpenMetadataWorkflowConfig.parse_obj(mock_tableau_config) + self.metabase = MetabaseSource.create( + mock_tableau_config["source"], + self.config.workflowConfig.openMetadataServerConfig, + ) + self.metabase.client = SimpleNamespace() + self.metabase.context.__dict__["dashboard_service"] = MOCK_DASHBOARD_SERVICE + + def test_dashboard_name(self): + assert ( + self.metabase.get_dashboard_name(MOCK_DASHBOARD_DETAILS) + == MOCK_DASHBOARD_DETAILS.name + ) + + def test_yield_chart(self): + """ + Function for testing charts + """ + chart_list = [] + results = self.metabase.yield_dashboard_chart(MOCK_DASHBOARD_DETAILS) + for result in results: + if isinstance(result, CreateChartRequest): + chart_list.append(result) + + for exptected, original in zip(EXPECTED_CHARTS, chart_list): + self.assertEqual(exptected, original) + + def test_yield_dashboard(self): + """ + Function for testing charts + """ + results = list(self.metabase.yield_dashboard(MOCK_DASHBOARD_DETAILS)) + self.assertEqual(EXPECTED_DASHBOARD, list(results)) + + @patch.object(fqn, "build", return_value=None) + @patch.object(OpenMetadata, "get_by_name", return_value=EXAMPLE_DASHBOARD) + @patch.object(MetabaseMetadata, "search_table_entities", return_value=EXAMPLE_TABLE) + def test_yield_lineage(self, *_): + """ + Function to test out lineage + """ + self.metabase.client.get_database = lambda *_: None + self.metabase.client.get_table = lambda *_: MetabaseTable( + schema="test_schema", display_name="test_table" + ) + + # if no db service name then no lineage generated + result = self.metabase.yield_dashboard_lineage_details( + dashboard_details=MOCK_DASHBOARD_DETAILS, db_service_name=None + ) + self.assertEqual(list(result), []) + + # test out _yield_lineage_from_api + mock_dashboard = deepcopy(MOCK_DASHBOARD_DETAILS) + mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[0]] + result = self.metabase.yield_dashboard_lineage_details( + dashboard_details=mock_dashboard, db_service_name="db.service.name" + ) + self.assertEqual(next(result), EXPECTED_LINEAGE) + + # test out _yield_lineage_from_query + mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[1]] + result = self.metabase.yield_dashboard_lineage_details( + dashboard_details=mock_dashboard, db_service_name="db.service.name" + ) + self.assertEqual(next(result), EXPECTED_LINEAGE) + + # test out if no query type + mock_dashboard.ordered_cards = [MOCK_DASHBOARD_DETAILS.ordered_cards[2]] + result = self.metabase.yield_dashboard_lineage_details( + dashboard_details=mock_dashboard, db_service_name="db.service.name" + ) + self.assertEqual(list(result), []) diff --git a/ingestion/tests/unit/topology/dashboard/test_quicksight.py b/ingestion/tests/unit/topology/dashboard/test_quicksight.py index fa10e2f81e9..7e45a5b982b 100644 --- a/ingestion/tests/unit/topology/dashboard/test_quicksight.py +++ b/ingestion/tests/unit/topology/dashboard/test_quicksight.py @@ -1,3 +1,14 @@ +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Test QuickSight using the topology """