yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-17 03:43:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest/mongodb): fix schema inference for lists of values (#9145)

Browse Source
This commit is contained in:
Harshal Sheth 2023-11-01 16:58:37 -07:00 committed by GitHub
parent 932eebea35
commit 50789224a1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
metadata-ingestion/src/datahub/ingestion/source/schema_inference/object.py
View File

@ -16,7 +16,7 @@ class SchemaDescription(BasicSchemaDescription):
nullable: bool # if field is ever missing nullable: bool # if field is ever missing
def is_field_nullable(doc: Dict[str, Any], field_path: Tuple) -> bool: def is_field_nullable(doc: Dict[str, Any], field_path: Tuple[str, ...]) -> bool:
""" """
Check if a nested field is nullable in a document from a collection. Check if a nested field is nullable in a document from a collection.
@ -54,7 +54,10 @@ def is_field_nullable(doc: Dict[str, Any], field_path: Tuple) -> bool:
# count empty lists of nested objects as nullable # count empty lists of nested objects as nullable
if len(value) == 0: if len(value) == 0:
return True return True
return any(is_field_nullable(x, remaining_fields) for x in doc[field]) return any(
isinstance(x, dict) and is_field_nullable(x, remaining_fields)
for x in doc[field]
)
# any other types to check? # any other types to check?
# raise ValueError("Nested type not 'list' or 'dict' encountered") # raise ValueError("Nested type not 'list' or 'dict' encountered")