From ed7f74c7b2cbc2101db36f8cb4a5f7dc1e34a581 Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:01:40 +0530 Subject: [PATCH] Improve exception handling for superset (#12753) * Improve exception handling for supersert * Update ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py --- .../source/dashboard/superset/api_source.py | 72 ++++++++++--------- .../source/dashboard/superset/client.py | 19 ++--- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py index aa525722b70..ddd923e7b95 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py @@ -81,23 +81,29 @@ class SupersetAPISource(SupersetSourceMixin): """ Method to Get Dashboard Entity """ - dashboard_request = CreateDashboardRequest( - name=dashboard_details.id, - displayName=dashboard_details.dashboard_title, - sourceUrl=f"{clean_uri(self.service_connection.hostPort)}{dashboard_details.url}", - charts=[ - fqn.build( - self.metadata, - entity_type=Chart, - service_name=self.context.dashboard_service.fullyQualifiedName.__root__, - chart_name=chart.name.__root__, - ) - for chart in self.context.charts - ], - service=self.context.dashboard_service.fullyQualifiedName.__root__, - ) - yield dashboard_request - self.register_record(dashboard_request=dashboard_request) + try: + dashboard_request = CreateDashboardRequest( + name=dashboard_details.id, + displayName=dashboard_details.dashboard_title, + sourceUrl=f"{clean_uri(self.service_connection.hostPort)}{dashboard_details.url}", + charts=[ + fqn.build( + self.metadata, + entity_type=Chart, + service_name=self.context.dashboard_service.fullyQualifiedName.__root__, + chart_name=chart.name.__root__, + ) + for chart in self.context.charts + ], + service=self.context.dashboard_service.fullyQualifiedName.__root__, + ) + yield dashboard_request + self.register_record(dashboard_request=dashboard_request) + except Exception as exc: # pylint: disable=broad-except + logger.debug(traceback.format_exc()) + logger.warning( + f"Error creating dashboard [{dashboard_details.dashboard_title}]: {exc}" + ) def _get_datasource_fqn_for_lineage( self, chart_json: ChartResult, db_service_entity: DatabaseService @@ -172,22 +178,24 @@ class SupersetAPISource(SupersetSourceMixin): if self.source_config.includeDataModels: for chart_id in self._get_charts_of_dashboard(dashboard_details): - chart_json = self.all_charts.get(chart_id) - if not chart_json: - logger.warning( - f"chart details for id: {chart_id} not found, skipped" - ) - continue - datasource_json = self.client.fetch_datasource(chart_json.datasource_id) - if filter_by_datamodel( - self.source_config.dataModelFilterPattern, - datasource_json.result.table_name, - ): - self.status.filter( - datasource_json.result.table_name, "Data model filtered out." - ) - try: + chart_json = self.all_charts.get(chart_id) + if not chart_json: + logger.warning( + f"chart details for id: {chart_id} not found, skipped" + ) + continue + datasource_json = self.client.fetch_datasource( + chart_json.datasource_id + ) + if filter_by_datamodel( + self.source_config.dataModelFilterPattern, + datasource_json.result.table_name, + ): + self.status.filter( + datasource_json.result.table_name, + "Data model filtered out.", + ) data_model_request = CreateDashboardDataModelRequest( name=datasource_json.id, displayName=datasource_json.result.table_name, diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py index 18055f51915..85c13e5f324 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/client.py @@ -111,7 +111,9 @@ class SupersetAPIClient: logger.warning("Failed to fetch the dashboard count") return 0 - def fetch_dashboards(self, current_page: int, page_size: int): + def fetch_dashboards( + self, current_page: int, page_size: int + ) -> SupersetDashboardCount: """ Fetch dashboards @@ -133,7 +135,7 @@ class SupersetAPIClient: except Exception: logger.debug(traceback.format_exc()) logger.warning("Failed to fetch the dashboard list") - return None + return SupersetDashboardCount() def fetch_total_charts(self) -> int: """ @@ -153,7 +155,7 @@ class SupersetAPIClient: logger.warning("Failed to fetch the chart count") return 0 - def fetch_charts(self, current_page: int, page_size: int): + def fetch_charts(self, current_page: int, page_size: int) -> SupersetChart: """ Fetch charts @@ -175,13 +177,13 @@ class SupersetAPIClient: except Exception: logger.debug(traceback.format_exc()) logger.warning("Failed to fetch the dashboard list") - return None + return SupersetChart() def fetch_charts_with_id(self, chart_id: str): response = self.client.get(f"/chart/{chart_id}") return response - def fetch_datasource(self, datasource_id: str): + def fetch_datasource(self, datasource_id: str) -> SupersetDatasource: """ Fetch data source @@ -199,9 +201,10 @@ class SupersetAPIClient: except Exception: logger.debug(traceback.format_exc()) logger.warning("Failed to fetch the dashboard list") - return None - def fetch_database(self, database_id: str): + return SupersetDatasource() + + def fetch_database(self, database_id: str) -> ListDatabaseResult: """ Fetch database @@ -219,4 +222,4 @@ class SupersetAPIClient: except Exception: logger.debug(traceback.format_exc()) logger.warning("Failed to fetch the database list") - return None + return ListDatabaseResult()