yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-10-31 10:49:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingestion): bigquery-usage - Collect stats from read event reasons (#5118)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-06-08 14:18:22 +02:00 committed by GitHub
parent 538cfba526
commit c677a06fd8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/usage/bigquery_usage.py
View File

@ -992,6 +992,11 @@ class BigQueryUsageSource(Source):
if not self._is_table_allowed(event.resource): if not self._is_table_allowed(event.resource):
self.report.num_filtered_read_events += 1 self.report.num_filtered_read_events += 1
continue continue
if event.readReason:
self.report.read_reasons_stat[event.readReason] = (
self.report.read_reasons_stat.get(event.readReason, 0) + 1
)
self.report.num_read_events += 1 self.report.num_read_events += 1
missing_query_entry = QueryEvent.get_missing_key_entry(entry) missing_query_entry = QueryEvent.get_missing_key_entry(entry)

3
metadata-ingestion/src/datahub/ingestion/source_report/usage/bigquery_usage.py
View File

@ -25,6 +25,9 @@ class BigQueryUsageSourceReport(SourceReport):
log_entry_end_time: Optional[str] = None log_entry_end_time: Optional[str] = None
num_usage_workunits_emitted: Optional[int] = None num_usage_workunits_emitted: Optional[int] = None
num_operational_stats_workunits_emitted: Optional[int] = None num_operational_stats_workunits_emitted: Optional[int] = None
read_reasons_stat: Counter[str] = dataclasses.field(
default_factory=collections.Counter
)
def report_dropped(self, key: str) -> None: def report_dropped(self, key: str) -> None:
self.dropped_table[key] += 1 self.dropped_table[key] += 1