yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-25 09:00:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): disable stateful_ingestion failsafe by default (#6537)

Browse Source
This commit is contained in:
Harshal Sheth 2022-11-23 14:51:56 -05:00 committed by GitHub
parent 8d525d67a9
commit 4db51b4c4f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py
View File

@ -34,10 +34,11 @@ class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig):
description="Soft-deletes the entities present in the last successful run but missing in the current run with stateful_ingestion enabled.",
)
fail_safe_threshold: float = pydantic.Field(
default=20.0,
default=100.0,
description="Prevents large amount of soft deletes & the state from committing from accidental changes to the source configuration if the relative change percent in entities compared to the previous state is above the 'fail_safe_threshold'.",
le=100.0, # mypy does not work with pydantic.confloat. This is the recommended work-around.
le=100.0,
ge=0.0,
hidden_from_schema=True,
)
@ -261,12 +262,16 @@ class StaleEntityRemovalHandler(
if (
entity_difference_percent
> self.stateful_ingestion_config.fail_safe_threshold
# Adding this check to protect against cases where get_percent_entities_changed returns over 100%.
# This previously happened due to a bug in the implementation, which caused this condition to be
# triggered too frequently.
and self.stateful_ingestion_config.fail_safe_threshold < 100.0
):
# Log the failure. This would prevent the current state from getting committed.
self.source.get_report().report_failure(
"Stateful Ingestion",
f"Fail safe mode triggered, entity difference percent:{entity_difference_percent}"
" > fail_safe_threshold:{self.stateful_ingestion_config.fail_safe_threshold}",
f"Will not soft-delete entities, since we'd be deleting {entity_difference_percent:.1f}% of the existing entities. "
f"To force a deletion, increase the value of 'stateful_ingestion.fail_safe_threshold' (currently {self.stateful_ingestion_config.fail_safe_threshold})",
)
# Bail so that we don't emit the stale entity removal workunits.
return