yujunjun/datahub
2
0
Fork 0
mirror of https://github.com/datahub-project/datahub.git synced 2025-11-18 20:57:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sdk): make Filter type permissive of implicit and dict (#14569)

Browse Source
This commit is contained in:
Mayuri Nehate 2025-08-28 15:22:00 +05:30 committed by GitHub
parent 340b1bf930
commit fe8f108746
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 47 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
metadata-ingestion/src/datahub/sdk/search_filters.py
View File

@ -384,6 +384,21 @@ def _filter_discriminator(v: Any) -> Optional[str]:
return None return None
def _parse_and_like_filter(value: Any) -> Any:
# Do not parse if filter is already of type and/or/not or a custom condition
# also do not parse container filter if direct_descendants_only is specified
if (
isinstance(value, dict)
and not set(value.keys()).intersection(
{"and", "or", "not", "field", "condition", "direct_descendants_only"}
)
and len(value) > 1
):
return {"and": [{k: v} for k, v in value.items()]}
return value
if TYPE_CHECKING or not PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR: if TYPE_CHECKING or not PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR:
# The `not TYPE_CHECKING` bit is required to make the linter happy, # The `not TYPE_CHECKING` bit is required to make the linter happy,
# since we currently only run mypy with pydantic v1. # since we currently only run mypy with pydantic v1.
@ -445,6 +460,7 @@ else:
], ],
Discriminator(_filter_discriminator), Discriminator(_filter_discriminator),
], ],
pydantic.BeforeValidator(_parse_and_like_filter),
pydantic.BeforeValidator(_parse_json_from_string), pydantic.BeforeValidator(_parse_json_from_string),
] ]

31
metadata-ingestion/tests/unit/sdk_v2/test_search_client.py
View File

@ -334,6 +334,12 @@ def test_tagged_union_error_messages() -> None:
): ):
load_filters({"and": [{"unknown_field": 6}]}) load_filters({"and": [{"unknown_field": 6}]})
@pytest.mark.skipif(
not PYDANTIC_SUPPORTS_CALLABLE_DISCRIMINATOR,
reason="Tagged union w/ callable discriminator is not supported by the current pydantic version",
)
def test_filter_before_validators() -> None:
# Test that we can load a filter from a string. # Test that we can load a filter from a string.
# Sometimes we get filters encoded as JSON, and we want to handle those gracefully. # Sometimes we get filters encoded as JSON, and we want to handle those gracefully.
filter_str = '{\n "and": [\n {"entity_type": ["dataset"]},\n {"entity_subtype": ["Table"]},\n {"platform": ["snowflake"]}\n ]\n}' filter_str = '{\n "and": [\n {"entity_type": ["dataset"]},\n {"entity_subtype": ["Table"]},\n {"platform": ["snowflake"]}\n ]\n}'
@ -350,6 +356,31 @@ def test_tagged_union_error_messages() -> None:
): ):
load_filters("this is invalid json but should not raise a json error") load_filters("this is invalid json but should not raise a json error")
# Test that we can load a filter from and-like dictionary.
# Sometimes we get filters that are not wrapped in an "and" clause.
filter_str = '{"entity_type": ["dataset"], "entity_subtype": ["Table"], "platform": ["snowflake"]}'
assert load_filters(filter_str) == F.and_(
F.entity_type("dataset"),
F.entity_subtype("Table"),
F.platform("snowflake"),
)
filter_str = '{"entity_type": ["dataset"], "container": ["urn:li:container:f784c48c306ba1c775ef917e2f8c1560"]}'
assert load_filters(filter_str) == F.and_(
F.entity_type("dataset"),
F.container("urn:li:container:f784c48c306ba1c775ef917e2f8c1560"),
)
filter_str = '{"entity_type": ["dataset"], "container": ["urn:li:container:f784c48c306ba1c775ef917e2f8c1560"], "direct_descendants_only": true}'
with pytest.raises(
ValidationError,
match=re.compile(
r"1 validation error.*container\.entity_type.*Extra inputs are not permitted.*",
re.DOTALL,
),
):
load_filters(filter_str)
def test_invalid_filter() -> None: def test_invalid_filter() -> None:
with pytest.raises(InvalidUrnError): with pytest.raises(InvalidUrnError):