2022-12-12 09:22:10 -05:00
|
|
|
import os
|
|
|
|
|
import pathlib
|
2023-02-07 09:09:34 -05:00
|
|
|
from unittest.mock import patch
|
|
|
|
|
|
2023-02-27 17:30:54 +01:00
|
|
|
import pytest
|
2023-02-07 09:09:34 -05:00
|
|
|
import requests
|
2023-03-23 20:14:57 -07:00
|
|
|
from requests.models import Response
|
2022-12-12 09:22:10 -05:00
|
|
|
|
2023-04-13 15:39:08 -04:00
|
|
|
from unstructured.documents.elements import PageBreak, Title
|
2022-12-12 09:22:10 -05:00
|
|
|
from unstructured.partition.html import partition_html
|
|
|
|
|
|
|
|
|
|
DIRECTORY = pathlib.Path(__file__).parent.resolve()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_from_filename():
|
2023-05-15 18:25:39 -04:00
|
|
|
directory = os.path.join(DIRECTORY, "..", "..", "example-docs")
|
|
|
|
|
filename = os.path.join(directory, "example-10k.html")
|
2022-12-12 09:22:10 -05:00
|
|
|
elements = partition_html(filename=filename)
|
2023-02-08 10:11:15 -05:00
|
|
|
assert PageBreak() not in elements
|
|
|
|
|
assert len(elements) > 0
|
2023-05-15 18:25:39 -04:00
|
|
|
assert elements[0].metadata.filename == "example-10k.html"
|
|
|
|
|
assert elements[0].metadata.file_directory == directory
|
2023-02-08 10:11:15 -05:00
|
|
|
|
|
|
|
|
|
2023-05-30 15:47:55 -05:00
|
|
|
def test_partition_html_from_filename_metadata_false():
|
|
|
|
|
directory = os.path.join(DIRECTORY, "..", "..", "example-docs")
|
|
|
|
|
filename = os.path.join(directory, "example-10k.html")
|
|
|
|
|
elements = partition_html(filename=filename, include_metadata=False)
|
|
|
|
|
metadata_present = any(element.metadata.to_dict() for element in elements)
|
|
|
|
|
assert not metadata_present
|
|
|
|
|
|
|
|
|
|
|
2023-02-08 10:11:15 -05:00
|
|
|
def test_partition_html_with_page_breaks():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
|
|
|
|
elements = partition_html(filename=filename, include_page_breaks=True)
|
|
|
|
|
assert PageBreak() in elements
|
2022-12-12 09:22:10 -05:00
|
|
|
assert len(elements) > 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_from_file():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
2023-02-27 17:30:54 +01:00
|
|
|
with open(filename) as f:
|
2022-12-12 09:22:10 -05:00
|
|
|
elements = partition_html(file=f)
|
|
|
|
|
assert len(elements) > 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_from_text():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
2023-02-27 17:30:54 +01:00
|
|
|
with open(filename) as f:
|
2022-12-12 09:22:10 -05:00
|
|
|
text = f.read()
|
|
|
|
|
elements = partition_html(text=text)
|
|
|
|
|
assert len(elements) > 0
|
|
|
|
|
|
|
|
|
|
|
2023-03-28 17:03:51 -04:00
|
|
|
def test_partition_html_from_text_works_with_empty_string():
|
|
|
|
|
assert partition_html(text="") == []
|
|
|
|
|
|
|
|
|
|
|
2023-02-07 09:09:34 -05:00
|
|
|
class MockResponse:
|
|
|
|
|
def __init__(self, text, status_code, headers={}):
|
|
|
|
|
self.text = text
|
|
|
|
|
self.status_code = status_code
|
|
|
|
|
self.ok = status_code < 300
|
|
|
|
|
self.headers = headers
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_from_url():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
2023-02-27 17:30:54 +01:00
|
|
|
with open(filename) as f:
|
2023-02-07 09:09:34 -05:00
|
|
|
text = f.read()
|
|
|
|
|
|
|
|
|
|
response = MockResponse(text=text, status_code=200, headers={"Content-Type": "text/html"})
|
|
|
|
|
with patch.object(requests, "get", return_value=response) as _:
|
|
|
|
|
elements = partition_html(url="https://fake.url")
|
|
|
|
|
|
|
|
|
|
assert len(elements) > 0
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_from_url_raises_with_bad_status_code():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
2023-02-27 17:30:54 +01:00
|
|
|
with open(filename) as f:
|
2023-02-07 09:09:34 -05:00
|
|
|
text = f.read()
|
|
|
|
|
|
|
|
|
|
response = MockResponse(text=text, status_code=500, headers={"Content-Type": "text/html"})
|
|
|
|
|
with patch.object(requests, "get", return_value=response) as _:
|
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
|
partition_html(url="https://fake.url")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_from_url_raises_with_bad_content_type():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
2023-02-27 17:30:54 +01:00
|
|
|
with open(filename) as f:
|
2023-02-07 09:09:34 -05:00
|
|
|
text = f.read()
|
|
|
|
|
|
|
|
|
|
response = MockResponse(
|
2023-02-27 17:30:54 +01:00
|
|
|
text=text,
|
|
|
|
|
status_code=200,
|
|
|
|
|
headers={"Content-Type": "application/json"},
|
2023-02-07 09:09:34 -05:00
|
|
|
)
|
|
|
|
|
with patch.object(requests, "get", return_value=response) as _:
|
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
|
partition_html(url="https://fake.url")
|
|
|
|
|
|
|
|
|
|
|
2023-03-23 20:14:57 -07:00
|
|
|
def test_partition_from_url_uses_headers(mocker):
|
|
|
|
|
test_url = "https://example.com"
|
|
|
|
|
test_headers = {"User-Agent": "test"}
|
|
|
|
|
|
|
|
|
|
response = Response()
|
|
|
|
|
response.status_code = 200
|
|
|
|
|
response._content = (
|
|
|
|
|
b"<html><head></head><body><p>What do i know? Who needs to know it?</p></body></html>"
|
|
|
|
|
)
|
|
|
|
|
response.headers = {"Content-Type": "text/html"}
|
|
|
|
|
|
|
|
|
|
mock_get = mocker.patch("requests.get", return_value=response)
|
|
|
|
|
|
|
|
|
|
partition_html(url=test_url, headers=test_headers)
|
|
|
|
|
|
|
|
|
|
# Check if requests.get was called with the correct arguments
|
2023-04-20 11:13:56 -04:00
|
|
|
mock_get.assert_called_once_with(test_url, headers=test_headers, verify=True)
|
2023-03-23 20:14:57 -07:00
|
|
|
|
|
|
|
|
|
2022-12-12 09:22:10 -05:00
|
|
|
def test_partition_html_raises_with_none_specified():
|
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
|
partition_html()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_raises_with_too_many_specified():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
2023-02-27 17:30:54 +01:00
|
|
|
with open(filename) as f:
|
2022-12-12 09:22:10 -05:00
|
|
|
text = f.read()
|
|
|
|
|
|
|
|
|
|
with pytest.raises(ValueError):
|
|
|
|
|
partition_html(filename=filename, text=text)
|
2023-03-02 14:03:13 -05:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_on_ideas_page():
|
|
|
|
|
filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "ideas-page.html")
|
|
|
|
|
elements = partition_html(filename=filename)
|
|
|
|
|
document_text = "\n\n".join([str(el) for el in elements])
|
|
|
|
|
assert document_text.startswith("January 2023(Someone fed my essays into GPT")
|
|
|
|
|
assert document_text.endswith("whole new fractal buds.")
|
2023-03-13 13:06:45 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_user_without_file_write_permission_can_partition_html(tmp_path, monkeypatch):
|
|
|
|
|
example_filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "example-10k.html")
|
|
|
|
|
|
|
|
|
|
# create a file with no write permissions
|
|
|
|
|
read_only_file = tmp_path / "example-10k-readonly.html"
|
|
|
|
|
read_only_file.touch()
|
|
|
|
|
|
|
|
|
|
# set content of read_only_file to be that of example-10k.html
|
|
|
|
|
with open(example_filename) as f:
|
|
|
|
|
read_only_file.write_text(f.read())
|
|
|
|
|
|
|
|
|
|
# set read_only_file to be read only
|
|
|
|
|
read_only_file.chmod(0o444)
|
|
|
|
|
|
|
|
|
|
# partition html should still work
|
|
|
|
|
elements = partition_html(filename=read_only_file.resolve())
|
|
|
|
|
assert len(elements) > 0
|
2023-04-05 16:18:54 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_partition_html_processes_chinese_chracters():
|
|
|
|
|
html_text = "<html><div><p>每日新闻</p></div></html>"
|
|
|
|
|
elements = partition_html(text=html_text)
|
|
|
|
|
assert elements[0].text == "每日新闻"
|
2023-04-13 15:39:08 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def test_emoji_appears_with_emoji_utf8_code():
|
|
|
|
|
html_text = """\n<html charset="utf-8"><p>Hello 😀</p></html>"""
|
|
|
|
|
elements = partition_html(text=html_text)
|
|
|
|
|
assert elements[0] == Title("Hello 😀")
|
