yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-12-26 17:37:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ingest): avrogen handling for missing fields with default values (#5844)

Browse Source
This commit is contained in:
Harshal Sheth 2022-09-08 14:05:28 -07:00 committed by GitHub
parent 2115d5bf1d
commit 6063484714
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 51 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
metadata-ingestion/setup.py
View File

@ -35,7 +35,7 @@ framework_common = {
"entrypoints",
"docker",
"expandvars>=0.6.5",
"avro-gen3==0.7.5",
"avro-gen3==0.7.6",
# "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3",
"avro>=1.10.2,<1.11",
"python-dateutil>=2.8.0",

50
metadata-ingestion/tests/unit/serde/test_serde.py
View File

@ -240,3 +240,53 @@ def test_null_hiding() -> None:
"recursive": False,
"type": {"type": {"com.linkedin.pegasus2avro.schema.StringType": {}}},
}
def test_missing_optional_simple() -> None:
original = models.DataHubResourceFilterClass.from_obj(
{
"allResources": False,
"filter": {
"criteria": [
{
"condition": "EQUALS",
"field": "RESOURCE_TYPE",
"values": ["notebook", "dataset", "dashboard"],
}
]
},
}
)
# This one is missing the optional filters.allResources field.
revised_obj = {
"filter": {
"criteria": [
{
"condition": "EQUALS",
"field": "RESOURCE_TYPE",
"values": ["notebook", "dataset", "dashboard"],
}
]
},
}
revised = models.DataHubResourceFilterClass.from_obj(revised_obj)
assert revised.validate()
assert original == revised
def test_missing_optional_in_union() -> None:
# This one doesn't contain any optional fields and should work fine.
revised_json = json.loads(
'{"lastUpdatedTimestamp":1662356745807,"actors":{"groups":[],"resourceOwners":false,"allUsers":true,"allGroups":false,"users":[]},"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"displayName":"customtest","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]},"allResources":false},"description":"","state":"ACTIVE","type":"METADATA"}'
)
revised = models.DataHubPolicyInfoClass.from_obj(revised_json)
# This one is missing the optional filters.allResources field.
original_json = json.loads(
'{"privileges":["EDIT_ENTITY_ASSERTIONS","EDIT_DATASET_COL_GLOSSARY_TERMS","EDIT_DATASET_COL_TAGS","EDIT_DATASET_COL_DESCRIPTION"],"actors":{"resourceOwners":false,"groups":[],"allGroups":false,"allUsers":true,"users":[]},"lastUpdatedTimestamp":1662356745807,"displayName":"customtest","description":"","resources":{"filter":{"criteria":[{"field":"RESOURCE_TYPE","condition":"EQUALS","values":["notebook","dataset","dashboard"]}]}},"state":"ACTIVE","type":"METADATA"}'
)
original = models.DataHubPolicyInfoClass.from_obj(original_json)
assert revised == original