yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-09-25 17:15:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): looker - handle invalid json in Looker query dynamic field (#5083)

Browse Source
This commit is contained in:
Tamas Nemeth 2022-06-03 23:30:57 +02:00 committed by GitHub
parent 59d33ca28b
commit aeffd26bb3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
metadata-ingestion/src/datahub/ingestion/source/looker.py
View File

@ -6,6 +6,7 @@ import os
import re import re
from dataclasses import dataclass from dataclasses import dataclass
from dataclasses import field as dataclass_field from dataclasses import field as dataclass_field
from json import JSONDecodeError
from typing import ( from typing import (
Any, Any,
Dict, Dict,
@ -378,9 +379,16 @@ class LookerDashboardSource(Source):
# - looker table calculations: https://docs.looker.com/exploring-data/using-table-calculations # - looker table calculations: https://docs.looker.com/exploring-data/using-table-calculations
# - looker custom measures: https://docs.looker.com/de/exploring-data/adding-fields/custom-measure # - looker custom measures: https://docs.looker.com/de/exploring-data/adding-fields/custom-measure
# - looker custom dimensions: https://docs.looker.com/exploring-data/adding-fields/custom-measure#creating_a_custom_dimension_using_a_looker_expression # - looker custom dimensions: https://docs.looker.com/exploring-data/adding-fields/custom-measure#creating_a_custom_dimension_using_a_looker_expression
dynamic_fields = json.loads( try:
query.dynamic_fields if query.dynamic_fields is not None else "[]" dynamic_fields = json.loads(
) query.dynamic_fields if query.dynamic_fields is not None else "[]"
)
except JSONDecodeError as e:
logger.warning(
f"Json load failed on loading dynamic field with error: {e}. The field value was: {query.dynamic_fields}"
)
dynamic_fields = "[]"
custom_field_to_underlying_field = {} custom_field_to_underlying_field = {}
for field in dynamic_fields: for field in dynamic_fields:
# Table calculations can only reference fields used in the fields section, so this will always be a subset of of the query.fields # Table calculations can only reference fields used in the fields section, so this will always be a subset of of the query.fields