yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-02 11:49:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/bigquery): update usage query, remove erroneous init (#7811)

Browse Source
This commit is contained in:
Mayuri Nehate 2023-04-15 02:08:50 +05:30 committed by GitHub
parent 37e7485184
commit 8ec74ce41c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/usage.py
View File

@ -218,7 +218,7 @@ class BigQueryUsageState(Closeable):
COUNT(q.query) query_count
FROM
read_events r
LEFT JOIN query_events q ON r.name = q.key
INNER JOIN query_events q ON r.name = q.key
GROUP BY r.timestamp, r.resource
) a
LEFT JOIN (
@ -231,7 +231,7 @@ class BigQueryUsageState(Closeable):
ROW_NUMBER() over (PARTITION BY r.timestamp, r.resource, q.query ORDER BY COUNT(r.key) DESC, q.query) as rank
FROM
read_events r
LEFT JOIN query_events q ON r.name = q.key
INNER JOIN query_events q ON r.name = q.key
GROUP BY r.timestamp, r.resource, q.query
ORDER BY r.timestamp, r.resource, query_count DESC, q.query
) WHERE rank <= {top_n}
@ -513,7 +513,6 @@ class BigQueryUsageExtractor:
def _get_bigquery_log_entries_via_gcp_logging(
self, client: GCPLoggingClient, limit: Optional[int] = None
) -> Iterable[AuditLogEntry]:
self.report.total_query_log_entries = 0
filter = self._generate_filter(BQ_AUDIT_V2)
logger.debug(filter)