yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-26 16:34:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/mode): Additional 404 handling and caching update (#13508)

Browse Source
This commit is contained in:
Michael Minichino 2025-05-14 09:09:40 -05:00 committed by GitHub
parent 5749f6f970
commit 8b4217f7fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 37 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
metadata-ingestion/src/datahub/ingestion/source/mode.py
View File

@ -1467,6 +1467,13 @@ class ModeSource(StatefulIngestionSourceBase):
) )
yield reports_page yield reports_page
except ModeRequestError as e: except ModeRequestError as e:
if isinstance(e, HTTPError) and e.response.status_code == 404:
self.report.report_warning(
title="No Reports Found in Space",
message="No reports were found in the space. It may have been recently deleted.",
context=f"Space Token: {space_token}, Error: {str(e)}",
)
else:
self.report.report_failure( self.report.report_failure(
title="Failed to Retrieve Reports for Space", title="Failed to Retrieve Reports for Space",
message="Unable to retrieve reports for space token.", message="Unable to retrieve reports for space token.",
@ -1490,10 +1497,17 @@ class ModeSource(StatefulIngestionSourceBase):
) )
yield dataset_page yield dataset_page
except ModeRequestError as e: except ModeRequestError as e:
if isinstance(e, HTTPError) and e.response.status_code == 404:
self.report.report_warning(
title="No Datasets Found in Space",
message="No datasets were found in the space. It may have been recently deleted.",
context=f"Space Token: {space_token}, Error: {str(e)}",
)
else:
self.report.report_failure( self.report.report_failure(
title="Failed to Retrieve Datasets for Space", title="Failed to Retrieve Datasets for Space",
message=f"Unable to retrieve datasets for space token {space_token}.", message=f"Unable to retrieve datasets for space token {space_token}.",
context=f"Error: {str(e)}", context=f"Space Token: {space_token}, Error: {str(e)}",
) )
def _get_queries(self, report_token: str) -> List[dict]: def _get_queries(self, report_token: str) -> List[dict]:
@ -1555,12 +1569,17 @@ class ModeSource(StatefulIngestionSourceBase):
) )
return charts.get("_embedded", {}).get("charts", []) return charts.get("_embedded", {}).get("charts", [])
except ModeRequestError as e: except ModeRequestError as e:
if isinstance(e, HTTPError) and e.response.status_code == 404:
self.report.report_warning(
title="No Charts Found for Query",
message="No charts were found for the query. The query may have been recently deleted.",
context=f"Report Token: {report_token}, Query Token: {query_token}, Error: {str(e)}",
)
else:
self.report.report_failure( self.report.report_failure(
title="Failed to Retrieve Charts", title="Failed to Retrieve Charts",
message="Unable to retrieve charts from Mode.", message="Unable to retrieve charts from Mode.",
context=f"Report Token: {report_token}, " context=f"Report Token: {report_token}, Query Token: {query_token}, Error: {str(e)}",
f"Query token: {query_token}, "
f"Error: {str(e)}",
) )
return [] return []
@ -1577,7 +1596,7 @@ class ModeSource(StatefulIngestionSourceBase):
yield data yield data
page += 1 page += 1
@lru_cache(maxsize=20480) @lru_cache(maxsize=None)
def _get_request_json(self, url: str) -> Dict: def _get_request_json(self, url: str) -> Dict:
r = tenacity.Retrying( r = tenacity.Retrying(
wait=wait_exponential( wait=wait_exponential(