yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-26 09:35:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ingest): report soft deleted stale entities with LossyList (#7907)

Browse Source
This commit is contained in:
Andrew Sikowitz 2023-04-27 18:40:19 -04:00 committed by GitHub
parent af09034523
commit ca3cab4e23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
metadata-ingestion/src/datahub/ingestion/source/state/stale_entity_removal_handler.py
View File

@ -30,6 +30,7 @@ from datahub.ingestion.source.state.use_case_handler import (
StatefulIngestionUsecaseHandlerBase, StatefulIngestionUsecaseHandlerBase,
) )
from datahub.metadata.schema_classes import StatusClass from datahub.metadata.schema_classes import StatusClass
from datahub.utilities.lossy_collections import LossyList
logger: logging.Logger = logging.getLogger(__name__) logger: logging.Logger = logging.getLogger(__name__)
@ -54,7 +55,7 @@ class StatefulStaleMetadataRemovalConfig(StatefulIngestionConfig):
@dataclass @dataclass
class StaleEntityRemovalSourceReport(StatefulIngestionReport): class StaleEntityRemovalSourceReport(StatefulIngestionReport):
soft_deleted_stale_entities: List[str] = field(default_factory=list) soft_deleted_stale_entities: LossyList[str] = field(default_factory=LossyList)
def report_stale_entity_soft_deleted(self, urn: str) -> None: def report_stale_entity_soft_deleted(self, urn: str) -> None:
self.soft_deleted_stale_entities.append(urn) self.soft_deleted_stale_entities.append(urn)