mirror of
https://github.com/Unstructured-IO/unstructured.git
synced 2025-07-03 23:20:35 +00:00
d726963e42
Each partitioner has a test like `test_partition_x_with_json()`. What these do is serialize the elements produced by the partitioner to JSON, then read them back in from JSON and compare the before and after elements. Because our element equality (`Element.__eq__()`) is shallow, this doesn't tell us a lot, but if we take it one more step, like `List[Element] -> JSON -> List[Element] -> JSON` and then compare the JSON, it gives us some confidence that the serialized elements can be "re-hydrated" without losing any information. This actually showed up a few problems, all in the serialization/deserialization (serde) code that all elements share.
153 lines
4.8 KiB
Python
153 lines
4.8 KiB
Python
from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
|
|
from unstructured.chunking.title import chunk_by_title
|
|
from unstructured.documents.elements import Title
|
|
from unstructured.partition.org import partition_org
|
|
|
|
|
|
def test_partition_org_from_filename(filename="example-docs/README.org"):
|
|
elements = partition_org(filename=filename)
|
|
|
|
assert elements[0] == Title("Example Docs")
|
|
assert elements[0].metadata.filetype == "text/org"
|
|
|
|
|
|
def test_partition_org_from_filename_with_metadata_filename(filename="example-docs/README.org"):
|
|
elements = partition_org(filename=filename, metadata_filename="test")
|
|
|
|
assert elements[0] == Title("Example Docs")
|
|
assert elements[0].metadata.filename == "test"
|
|
|
|
|
|
def test_partition_org_from_file(filename="example-docs/README.org"):
|
|
with open(filename, "rb") as f:
|
|
elements = partition_org(file=f)
|
|
|
|
assert elements[0] == Title("Example Docs")
|
|
assert elements[0].metadata.filetype == "text/org"
|
|
|
|
|
|
def test_partition_org_from_file_with_metadata_filename(filename="example-docs/README.org"):
|
|
with open(filename, "rb") as f:
|
|
elements = partition_org(file=f, metadata_filename="test")
|
|
|
|
assert elements[0] == Title("Example Docs")
|
|
assert elements[0].metadata.filename == "test"
|
|
|
|
|
|
def test_partition_org_from_filename_exclude_metadata(filename="example-docs/README.org"):
|
|
elements = partition_org(filename=filename, include_metadata=False)
|
|
|
|
for i in range(len(elements)):
|
|
assert elements[i].metadata.to_dict() == {}
|
|
|
|
|
|
def test_partition_org_from_file_exclude_metadata(filename="example-docs/README.org"):
|
|
with open(filename, "rb") as f:
|
|
elements = partition_org(file=f, include_metadata=False)
|
|
|
|
for i in range(len(elements)):
|
|
assert elements[i].metadata.to_dict() == {}
|
|
|
|
|
|
def test_partition_org_metadata_date(
|
|
mocker,
|
|
filename="example-docs/README.org",
|
|
):
|
|
mocked_last_modification_date = "2029-07-05T09:24:28"
|
|
|
|
mocker.patch(
|
|
"unstructured.partition.html.get_last_modified_date",
|
|
return_value=mocked_last_modification_date,
|
|
)
|
|
|
|
elements = partition_org(
|
|
filename=filename,
|
|
)
|
|
|
|
assert elements[0].metadata.last_modified == mocked_last_modification_date
|
|
|
|
|
|
def test_partition_org_with_custom_metadata_date(
|
|
mocker,
|
|
filename="example-docs/README.org",
|
|
):
|
|
mocked_last_modification_date = "2029-07-05T09:24:28"
|
|
expected_last_modification_date = "2020-07-05T09:24:28"
|
|
|
|
mocker.patch(
|
|
"unstructured.partition.html.get_last_modified_date",
|
|
return_value=mocked_last_modification_date,
|
|
)
|
|
|
|
elements = partition_org(
|
|
filename=filename,
|
|
metadata_last_modified=expected_last_modification_date,
|
|
)
|
|
|
|
assert elements[0].metadata.last_modified == expected_last_modification_date
|
|
|
|
|
|
def test_partition_org_from_file_metadata_date(
|
|
mocker,
|
|
filename="example-docs/README.org",
|
|
):
|
|
mocked_last_modification_date = "2029-07-05T09:24:28"
|
|
|
|
mocker.patch(
|
|
"unstructured.partition.html.get_last_modified_date_from_file",
|
|
return_value=mocked_last_modification_date,
|
|
)
|
|
|
|
with open(filename, "rb") as f:
|
|
elements = partition_org(
|
|
file=f,
|
|
)
|
|
|
|
assert elements[0].metadata.last_modified == mocked_last_modification_date
|
|
|
|
|
|
def test_partition_org_from_file_with_custom_metadata_date(
|
|
mocker,
|
|
filename="example-docs/README.org",
|
|
):
|
|
mocked_last_modification_date = "2029-07-05T09:24:28"
|
|
expected_last_modification_date = "2020-07-05T09:24:28"
|
|
|
|
mocker.patch(
|
|
"unstructured.partition.html.get_last_modified_date_from_file",
|
|
return_value=mocked_last_modification_date,
|
|
)
|
|
|
|
with open(filename, "rb") as f:
|
|
elements = partition_org(file=f, metadata_last_modified=expected_last_modification_date)
|
|
|
|
assert elements[0].metadata.last_modified == expected_last_modification_date
|
|
|
|
|
|
def test_partition_org_with_json():
|
|
elements = partition_org(example_doc_path("README.org"))
|
|
assert_round_trips_through_JSON(elements)
|
|
|
|
|
|
def test_add_chunking_strategy_by_title_on_partition_org(
|
|
filename="example-docs/README.org",
|
|
):
|
|
elements = partition_org(filename=filename)
|
|
chunk_elements = partition_org(filename, chunking_strategy="by_title")
|
|
chunks = chunk_by_title(elements)
|
|
assert chunk_elements != elements
|
|
assert chunk_elements == chunks
|
|
|
|
|
|
def test_partition_org_element_metadata_has_languages():
|
|
filename = "example-docs/README.org"
|
|
elements = partition_org(filename=filename)
|
|
assert elements[0].metadata.languages == ["eng"]
|
|
|
|
|
|
def test_partition_org_respects_detect_language_per_element():
|
|
filename = "example-docs/language-docs/eng_spa_mult.org"
|
|
elements = partition_org(filename=filename, detect_language_per_element=True)
|
|
langs = [element.metadata.languages for element in elements]
|
|
assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]
|