yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-27 10:04:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): add time taken by compute stats overall (#14713)

Browse Source
This commit is contained in:
Aseem Bansal 2025-09-09 14:53:13 +05:30 committed by GitHub
parent aee04b5680
commit 5ce93f54b4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
metadata-ingestion/src/datahub/ingestion/api/report.py
View File

@ -204,6 +204,7 @@ class ExamplesReport(Report, Closeable):
samples: Dict[str, Dict[str, List[str]]] = field( samples: Dict[str, Dict[str, List[str]]] = field(
default_factory=lambda: defaultdict(lambda: defaultdict(list)) default_factory=lambda: defaultdict(lambda: defaultdict(list))
) )
compute_stats_time_seconds: float = 0.0
_file_based_dict: Optional[FileBackedDict[SourceReportSubtypes]] = None _file_based_dict: Optional[FileBackedDict[SourceReportSubtypes]] = None
# We are adding this to make querying easier for fine-grained lineage # We are adding this to make querying easier for fine-grained lineage
@ -405,6 +406,7 @@ class ExamplesReport(Report, Closeable):
self._update_file_based_dict(urn, entityType, aspectName, mcp) self._update_file_based_dict(urn, entityType, aspectName, mcp)
def compute_stats(self) -> None: def compute_stats(self) -> None:
start_time = datetime.now()
if self._file_based_dict is None: if self._file_based_dict is None:
return return
@ -466,6 +468,8 @@ class ExamplesReport(Report, Closeable):
list(self._lineage_aspects_seen), "lineage" list(self._lineage_aspects_seen), "lineage"
) )
self._collect_samples_with_all_conditions("all_3") self._collect_samples_with_all_conditions("all_3")
end_time = datetime.now()
self.compute_stats_time_seconds += (end_time - start_time).total_seconds()
class EntityFilterReport(ReportAttribute): class EntityFilterReport(ReportAttribute):