yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-01 03:09:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/bigquery): ignore complex types from profiling (#6613)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-12-02 19:26:53 +01:00 committed by GitHub
parent 779d1cf369
commit 43775ecd49
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py
View File

@ -165,6 +165,16 @@ WHERE
continue
for table in tables[project][dataset]:
for column in table.columns:
# Profiler has issues with complex types (array, struct, geography, json), so we deny those types from profiling
# We also filter columns without data type as it means that column is part of a complex type.
if not column.data_type or any(
word in column.data_type.lower()
for word in ["array", "struct", "geography", "json"]
):
self.config.profile_pattern.deny.append(
f"^{project}.{dataset}.{table.name}.{column.field_path}$"
)
# Emit the profile work unit
profile_request = self.get_bigquery_profile_request(
project=project, dataset=dataset, table=table