yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-13 17:59:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/bigquery): Enable lineage and usage ingestion without tables (#7820)

Browse Source
This commit is contained in:
Andrew Sikowitz 2023-04-14 04:41:00 -04:00 committed by GitHub
parent d8d8176b1a
commit 408cd7db2a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
View File

@ -748,6 +748,25 @@ class BigqueryV2Source(StatefulIngestionSourceBase, TestableSource):
project_id=project_id, project_id=project_id,
dataset_name=dataset_name, dataset_name=dataset_name,
) )
elif self.config.include_table_lineage or self.config.include_usage_statistics:
# Need table_refs to calculate lineage and usage
for table_item in conn.list_tables(f"{project_id}.{dataset_name}"):
identifier = BigqueryTableIdentifier(
project_id=project_id,
dataset=dataset_name,
table=table_item.table_id,
)
if not self.config.table_pattern.allowed(identifier.raw_table_name()):
self.report.report_dropped(identifier.raw_table_name())
continue
try:
self.table_refs.add(
str(BigQueryTableRef(identifier).get_sanitized_table_ref())
)
except Exception as e:
logger.warning(
f"Could not create table ref for {table_item.path}: {e}"
)
if self.config.include_views: if self.config.include_views:
db_views[dataset_name] = list( db_views[dataset_name] = list(