yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-17 11:58:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): more error handling (#11969)

Browse Source
This commit is contained in:
Aseem Bansal 2024-11-27 16:49:01 +05:30 committed by GitHub
parent b6ccb8c9bf
commit 48d711b198
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 25 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
metadata-ingestion/src/datahub/ingestion/source/gc/datahub_gc.py
View File

@ -144,15 +144,32 @@ class DataHubGcSource(Source):
self,
) -> Iterable[MetadataWorkUnit]:
if self.config.cleanup_expired_tokens:
self.revoke_expired_tokens()
try:
self.revoke_expired_tokens()
except Exception as e:
self.report.failure("While trying to cleanup expired token ", exc=e)
if self.config.truncate_indices:
self.truncate_indices()
try:
self.truncate_indices()
except Exception as e:
self.report.failure("While trying to truncate indices ", exc=e)
if self.dataprocess_cleanup:
yield from self.dataprocess_cleanup.get_workunits_internal()
try:
yield from self.dataprocess_cleanup.get_workunits_internal()
except Exception as e:
self.report.failure("While trying to cleanup data process ", exc=e)
if self.soft_deleted_entities_cleanup:
self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities()
try:
self.soft_deleted_entities_cleanup.cleanup_soft_deleted_entities()
except Exception as e:
self.report.failure(
"While trying to cleanup soft deleted entities ", exc=e
)
if self.execution_request_cleanup:
self.execution_request_cleanup.run()
try:
self.execution_request_cleanup.run()
except Exception as e:
self.report.failure("While trying to cleanup execution request ", exc=e)
yield from []
def truncate_indices(self) -> None:

4
metadata-ingestion/src/datahub/ingestion/source/gc/dataprocess_cleanup.py
View File

@ -404,7 +404,9 @@ class DataProcessCleanup:
try:
self.delete_dpi_from_datajobs(datajob_entity)
except Exception as e:
logger.error(f"While trying to delete {datajob_entity} got {e}")
self.report.failure(
f"While trying to delete {datajob_entity} ", exc=e
)
if (
datajob_entity.total_runs == 0
and self.config.delete_empty_data_jobs