From 35d4c64e69242f7078a164899b9bf002c3592cdf Mon Sep 17 00:00:00 2001 From: Mayur Singal <39544459+ulixius9@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:33:59 +0530 Subject: [PATCH] Minor: Improve exception handling in Superset (#14371) --- .../source/dashboard/superset/api_source.py | 39 ++++--- .../source/dashboard/superset/db_source.py | 108 ++++++++++++------ .../source/dashboard/superset/mixin.py | 40 ++++--- 3 files changed, 120 insertions(+), 67 deletions(-) diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py index 66b06a46046..9802383aea5 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py @@ -123,19 +123,30 @@ class SupersetAPISource(SupersetSourceMixin): ) -> Iterable[Either[CreateChartRequest]]: """Method to fetch charts linked to dashboard""" for chart_id in self._get_charts_of_dashboard(dashboard_details): - chart_json = self.all_charts.get(chart_id) - if not chart_json: - logger.warning(f"chart details for id: {chart_id} not found, skipped") - continue - chart = CreateChartRequest( - name=chart_json.id, - displayName=chart_json.slice_name, - description=chart_json.description, - chartType=get_standard_chart_type(chart_json.viz_type), - sourceUrl=f"{clean_uri(self.service_connection.hostPort)}{chart_json.url}", - service=self.context.dashboard_service, - ) - yield Either(right=chart) + try: + chart_json = self.all_charts.get(chart_id) + if not chart_json: + logger.warning( + f"chart details for id: {chart_id} not found, skipped" + ) + continue + chart = CreateChartRequest( + name=chart_json.id, + displayName=chart_json.slice_name, + description=chart_json.description, + chartType=get_standard_chart_type(chart_json.viz_type), + sourceUrl=f"{clean_uri(self.service_connection.hostPort)}{chart_json.url}", + service=self.context.dashboard_service, + ) + yield Either(right=chart) + except Exception as exc: # pylint: disable=broad-except + yield Either( + left=StackTraceError( + name=chart_json.id, + error=f"Error creating chart [{chart_json.id} - {chart_json.slice_name}]: {exc}", + stack_trace=traceback.format_exc(), + ) + ) def _get_datasource_fqn( self, datasource_id: str, db_service_entity: DatabaseService @@ -166,7 +177,7 @@ class SupersetAPISource(SupersetSourceMixin): service_name=db_service_entity.name.__root__, ) return dataset_fqn - except KeyError as err: + except Exception as err: logger.debug(traceback.format_exc()) logger.warning( f"Failed to fetch Datasource with id [{datasource_id}]: {err}" diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py index 49e10dca33a..afe2f73ffb2 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/db_source.py @@ -71,15 +71,27 @@ class SupersetDBSource(SupersetSourceMixin): this step is done because fetch_total_charts api fetches all the required information which is not available in fetch_charts_with_id api """ - charts = self.engine.execute(FETCH_ALL_CHARTS) - for chart in charts: - chart_detail = FetchChart(**chart) - self.all_charts[chart_detail.id] = chart_detail + try: + charts = self.engine.execute(FETCH_ALL_CHARTS) + for chart in charts: + chart_detail = FetchChart(**chart) + self.all_charts[chart_detail.id] = chart_detail + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning(f"Failed to fetch chart list due to - {err}]") def get_column_list(self, table_name: str) -> Iterable[FetchChart]: - sql_query = sql.text(FETCH_COLUMN.format(table_name=table_name.lower())) - col_list = self.engine.execute(sql_query) - return [FetchColumn(**col) for col in col_list] + try: + if table_name: + sql_query = sql.text(FETCH_COLUMN.format(table_name=table_name.lower())) + col_list = self.engine.execute(sql_query) + return [FetchColumn(**col) for col in col_list] + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Failed to fetch column name list for table: [{table_name} due to - {err}]" + ) + return [] def get_dashboards_list(self) -> Iterable[FetchDashboard]: """ @@ -93,23 +105,35 @@ class SupersetDBSource(SupersetSourceMixin): self, dashboard_details: FetchDashboard ) -> Iterable[Either[CreateDashboardRequest]]: """Method to Get Dashboard Entity""" - dashboard_request = CreateDashboardRequest( - name=dashboard_details.id, - displayName=dashboard_details.dashboard_title, - sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/superset/dashboard/{dashboard_details.id}/", - charts=[ - fqn.build( - self.metadata, - entity_type=Chart, - service_name=self.context.dashboard_service, - chart_name=chart, + try: + dashboard_request = CreateDashboardRequest( + name=dashboard_details.id, + displayName=dashboard_details.dashboard_title, + sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/superset/dashboard/{dashboard_details.id}/", + charts=[ + fqn.build( + self.metadata, + entity_type=Chart, + service_name=self.context.dashboard_service, + chart_name=chart, + ) + for chart in self.context.charts + ], + service=self.context.dashboard_service, + ) + yield Either(right=dashboard_request) + self.register_record(dashboard_request=dashboard_request) + except Exception as exc: + yield Either( + left=StackTraceError( + name=dashboard_details.id, + error=( + f"Error yielding Dashboard [{dashboard_details.id} " + f"- {dashboard_details.dashboard_title}]: {exc}" + ), + stack_trace=traceback.format_exc(), ) - for chart in self.context.charts - ], - service=self.context.dashboard_service, - ) - yield Either(right=dashboard_request) - self.register_record(dashboard_request=dashboard_request) + ) def _get_datasource_fqn_for_lineage( self, chart_json: FetchChart, db_service_entity: DatabaseService @@ -127,19 +151,31 @@ class SupersetDBSource(SupersetSourceMixin): Metod to fetch charts linked to dashboard """ for chart_id in self._get_charts_of_dashboard(dashboard_details): - chart_json = self.all_charts.get(chart_id) - if not chart_json: - logger.warning(f"chart details for id: {chart_id} not found, skipped") - continue - chart = CreateChartRequest( - name=chart_json.id, - displayName=chart_json.slice_name, - description=chart_json.description, - chartType=get_standard_chart_type(chart_json.viz_type), - sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/explore/?slice_id={chart_json.id}", - service=self.context.dashboard_service, - ) - yield Either(right=chart) + try: + chart_json = self.all_charts.get(chart_id) + if not chart_json: + logger.warning( + f"chart details for id: {chart_id} not found, skipped" + ) + + continue + chart = CreateChartRequest( + name=chart_json.id, + displayName=chart_json.slice_name, + description=chart_json.description, + chartType=get_standard_chart_type(chart_json.viz_type), + sourceUrl=f"{clean_uri(self.service_connection.hostPort)}/explore/?slice_id={chart_json.id}", + service=self.context.dashboard_service, + ) + yield Either(right=chart) + except Exception as exc: + yield Either( + left=StackTraceError( + name=chart_json.id, + error=f"Error yielding Chart [{chart_json.id} - {chart_json.slice_name}]: {exc}", + stack_trace=traceback.format_exc(), + ) + ) def _get_database_name( self, sqa_str: str, db_service_entity: DatabaseService diff --git a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py index f77c374a70e..b37cf67c96c 100644 --- a/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py +++ b/ingestion/src/metadata/ingestion/source/dashboard/superset/mixin.py @@ -121,14 +121,20 @@ class SupersetSourceMixin(DashboardServiceSource): """ Method to fetch chart ids linked to dashboard """ - raw_position_data = dashboard_details.position_json - if raw_position_data: - position_data = json.loads(raw_position_data) - return [ - value.get("meta", {}).get("chartId") - for key, value in position_data.items() - if key.startswith("CHART-") and value.get("meta", {}).get("chartId") - ] + try: + raw_position_data = dashboard_details.position_json + if raw_position_data: + position_data = json.loads(raw_position_data) + return [ + value.get("meta", {}).get("chartId") + for key, value in position_data.items() + if key.startswith("CHART-") and value.get("meta", {}).get("chartId") + ] + except Exception as err: + logger.debug(traceback.format_exc()) + logger.warning( + f"Failed to charts of dashboard {dashboard_details.id} due to {err}" + ) return [] def yield_dashboard_lineage_details( @@ -146,16 +152,16 @@ class SupersetSourceMixin(DashboardServiceSource): for chart_id in self._get_charts_of_dashboard(dashboard_details): chart_json = self.all_charts.get(chart_id) if chart_json: - datasource_fqn = self._get_datasource_fqn_for_lineage( - chart_json, db_service_entity - ) - if not datasource_fqn: - continue - from_entity = self.metadata.get_by_name( - entity=Table, - fqn=datasource_fqn, - ) try: + datasource_fqn = self._get_datasource_fqn_for_lineage( + chart_json, db_service_entity + ) + if not datasource_fqn: + continue + from_entity = self.metadata.get_by_name( + entity=Table, + fqn=datasource_fqn, + ) datamodel_fqn = fqn.build( self.metadata, entity_type=DashboardDataModel,