yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-09 16:03:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest) - bigquery-beta - Using table ref instead of table id (#6193)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-10-13 10:03:33 +02:00 committed by GitHub
parent 6e34cd6001
commit 0162074a37
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 1 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py
View File

@ -234,9 +234,7 @@ timestamp < "{end_time}"
logger.info( logger.info(
f"Start iterating over log entries from BigQuery for {client.project}" f"Start iterating over log entries from BigQuery for {client.project}"
) )
for entry in entries: for entry in entries:
# for num in range(0, 100):
self.report.num_total_log_entries[client.project] += 1 self.report.num_total_log_entries[client.project] += 1
if self.report.num_total_log_entries[client.project] % 1000 == 0: if self.report.num_total_log_entries[client.project] % 1000 == 0:
logger.info( logger.info(
@ -418,9 +416,7 @@ timestamp < "{end_time}"
continue continue
has_table = False has_table = False
for ref_table in e.referencedTables: for ref_table in e.referencedTables:
ref_table_str = ( ref_table_str = str(ref_table.get_sanitized_table_ref())
ref_table.get_sanitized_table_ref().table_identifier.get_table_name()
)
if ref_table_str != destination_table_str: if ref_table_str != destination_table_str:
lineage_map[destination_table_str].add(ref_table_str) lineage_map[destination_table_str].add(ref_table_str)
has_table = True has_table = True