yujunjun/OpenMetadata
1
0
Fork 0
mirror of https://github.com/open-metadata/OpenMetadata.git synced 2026-01-07 04:56:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve exception handling for superset (#12753)

Browse Source 
* Improve exception handling for supersert

* Update ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py
This commit is contained in:
Mayur Singal 2023-08-04 15:01:40 +05:30 committed by GitHub
parent 2f3b965f1c
commit ed7f74c7b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 51 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
ingestion/src/metadata/ingestion/source/dashboard/superset/api_source.py
View File

@ -81,23 +81,29 @@ class SupersetAPISource(SupersetSourceMixin):
"""
Method to Get Dashboard Entity
"""
dashboard_request = CreateDashboardRequest(
name=dashboard_details.id,
displayName=dashboard_details.dashboard_title,
sourceUrl=f"{clean_uri(self.service_connection.hostPort)}{dashboard_details.url}",
charts=[
fqn.build(
self.metadata,
entity_type=Chart,
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
chart_name=chart.name.__root__,
)
for chart in self.context.charts
],
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
yield dashboard_request
self.register_record(dashboard_request=dashboard_request)
try:
dashboard_request = CreateDashboardRequest(
name=dashboard_details.id,
displayName=dashboard_details.dashboard_title,
sourceUrl=f"{clean_uri(self.service_connection.hostPort)}{dashboard_details.url}",
charts=[
fqn.build(
self.metadata,
entity_type=Chart,
service_name=self.context.dashboard_service.fullyQualifiedName.__root__,
chart_name=chart.name.__root__,
)
for chart in self.context.charts
],
service=self.context.dashboard_service.fullyQualifiedName.__root__,
)
yield dashboard_request
self.register_record(dashboard_request=dashboard_request)
except Exception as exc: # pylint: disable=broad-except
logger.debug(traceback.format_exc())
logger.warning(
f"Error creating dashboard [{dashboard_details.dashboard_title}]: {exc}"
)
def _get_datasource_fqn_for_lineage(
self, chart_json: ChartResult, db_service_entity: DatabaseService
@ -172,22 +178,24 @@ class SupersetAPISource(SupersetSourceMixin):
if self.source_config.includeDataModels:
for chart_id in self._get_charts_of_dashboard(dashboard_details):
chart_json = self.all_charts.get(chart_id)
if not chart_json:
logger.warning(
f"chart details for id: {chart_id} not found, skipped"
)
continue
datasource_json = self.client.fetch_datasource(chart_json.datasource_id)
if filter_by_datamodel(
self.source_config.dataModelFilterPattern,
datasource_json.result.table_name,
):
self.status.filter(
datasource_json.result.table_name, "Data model filtered out."
)
try:
chart_json = self.all_charts.get(chart_id)
if not chart_json:
logger.warning(
f"chart details for id: {chart_id} not found, skipped"
)
continue
datasource_json = self.client.fetch_datasource(
chart_json.datasource_id
)
if filter_by_datamodel(
self.source_config.dataModelFilterPattern,
datasource_json.result.table_name,
):
self.status.filter(
datasource_json.result.table_name,
"Data model filtered out.",
)
data_model_request = CreateDashboardDataModelRequest(
name=datasource_json.id,
displayName=datasource_json.result.table_name,

19
ingestion/src/metadata/ingestion/source/dashboard/superset/client.py
View File

@ -111,7 +111,9 @@ class SupersetAPIClient:
logger.warning("Failed to fetch the dashboard count")
return 0
def fetch_dashboards(self, current_page: int, page_size: int):
def fetch_dashboards(
self, current_page: int, page_size: int
) -> SupersetDashboardCount:
"""
Fetch dashboards
@ -133,7 +135,7 @@ class SupersetAPIClient:
except Exception:
logger.debug(traceback.format_exc())
logger.warning("Failed to fetch the dashboard list")
return None
return SupersetDashboardCount()
def fetch_total_charts(self) -> int:
"""
@ -153,7 +155,7 @@ class SupersetAPIClient:
logger.warning("Failed to fetch the chart count")
return 0
def fetch_charts(self, current_page: int, page_size: int):
def fetch_charts(self, current_page: int, page_size: int) -> SupersetChart:
"""
Fetch charts
@ -175,13 +177,13 @@ class SupersetAPIClient:
except Exception:
logger.debug(traceback.format_exc())
logger.warning("Failed to fetch the dashboard list")
return None
return SupersetChart()
def fetch_charts_with_id(self, chart_id: str):
response = self.client.get(f"/chart/{chart_id}")
return response
def fetch_datasource(self, datasource_id: str):
def fetch_datasource(self, datasource_id: str) -> SupersetDatasource:
"""
Fetch data source
@ -199,9 +201,10 @@ class SupersetAPIClient:
except Exception:
logger.debug(traceback.format_exc())
logger.warning("Failed to fetch the dashboard list")
return None
def fetch_database(self, database_id: str):
return SupersetDatasource()
def fetch_database(self, database_id: str) -> ListDatabaseResult:
"""
Fetch database
@ -219,4 +222,4 @@ class SupersetAPIClient:
except Exception:
logger.debug(traceback.format_exc())
logger.warning("Failed to fetch the database list")
return None
return ListDatabaseResult()