yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-11 08:52:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ingest): bigquery - support snapshot and partition tables during ingest & lineage (#3695)

Browse Source
This commit is contained in:
Gabe Lyons 2021-12-08 16:07:21 -08:00 committed by GitHub
parent 1d7ec8dba8
commit 3cc4e76748
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
metadata-ingestion/src/datahub/ingestion/source/sql/bigquery.py
View File

@ -187,8 +187,8 @@ class BigQuerySource(SQLAlchemySource):
): ):
continue continue
for ref_table in e.referencedTables: for ref_table in e.referencedTables:
destination_table_str = str(e.destinationTable) destination_table_str = str(e.destinationTable.remove_extras())
ref_table_str = str(ref_table) ref_table_str = str(ref_table.remove_extras())
if ref_table_str != destination_table_str: if ref_table_str != destination_table_str:
lineage_map[destination_table_str].add(ref_table_str) lineage_map[destination_table_str].add(ref_table_str)
return lineage_map return lineage_map
@ -294,7 +294,14 @@ class BigQuerySource(SQLAlchemySource):
) -> str: ) -> str:
assert inspector assert inspector
project_id = self._get_project_id(inspector) project_id = self._get_project_id(inspector)
return f"{project_id}.{schema}.{entity}" trimmed_table_name = (
BigQueryTableRef.from_spec_obj(
{"projectId": project_id, "datasetId": schema, "tableId": entity}
)
.remove_extras()
.table
)
return f"{project_id}.{schema}.{trimmed_table_name}"
def standardize_schema_table_names( def standardize_schema_table_names(
self, schema: str, entity: str self, schema: str, entity: str