From fd02f711cda7b78b052eca37d73baa399f1b55b0 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Thu, 8 Jul 2021 17:10:11 -0700 Subject: [PATCH] fix(ingest): handle 'fields' list missing in bigquery-usage (#2844) --- .../src/datahub/ingestion/source/bigquery_usage.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_usage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_usage.py index 3d01e23a1c..6ec9700d81 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_usage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_usage.py @@ -120,7 +120,7 @@ class ReadEvent: resource: BigQueryTableRef fieldsRead: List[str] - readReason: str + readReason: Optional[str] jobName: Optional[str] payload: Any @@ -143,8 +143,8 @@ class ReadEvent: resourceName = entry.payload["resourceName"] readInfo = entry.payload["metadata"]["tableDataRead"] - fields = readInfo["fields"] - readReason = readInfo["reason"] + fields = readInfo.get("fields", []) + readReason = readInfo.get("reason") jobName = None if readReason == "JOB": jobName = readInfo["jobName"] @@ -284,7 +284,7 @@ class BigQueryUsageSource(Source): self, entries: Iterable[AuditLogEntry] ) -> Iterable[Union[ReadEvent, QueryEvent]]: for entry in entries: - event: Union[ReadEvent, QueryEvent] + event: Union[None, ReadEvent, QueryEvent] = None if ReadEvent.can_parse_entry(entry): event = ReadEvent.from_entry(entry) elif QueryEvent.can_parse_entry(entry): @@ -294,7 +294,8 @@ class BigQueryUsageSource(Source): f"{entry.log_name}-{entry.insert_id}", f"unable to parse log entry: {entry!r}", ) - yield event + if event: + yield event def _join_events_by_job_id( self, events: Iterable[Union[ReadEvent, QueryEvent]]