2020-10-15 10:42:08 +02:00
|
|
|
from pathlib import Path
|
|
|
|
|
2022-03-09 13:46:47 +01:00
|
|
|
from haystack import Document
|
2022-01-26 18:12:55 +01:00
|
|
|
from haystack.nodes.file_converter.pdf import PDFToTextConverter
|
|
|
|
from haystack.nodes.preprocessor.preprocessor import PreProcessor
|
2020-10-30 18:06:02 +01:00
|
|
|
|
2022-05-17 10:55:53 +02:00
|
|
|
from ..conftest import SAMPLES_PATH
|
2020-10-15 10:42:08 +02:00
|
|
|
|
|
|
|
TEXT = """
|
|
|
|
This is a sample sentence in paragraph_1. This is a sample sentence in paragraph_1. This is a sample sentence in
|
|
|
|
paragraph_1. This is a sample sentence in paragraph_1. This is a sample sentence in paragraph_1.
|
|
|
|
|
|
|
|
This is a sample sentence in paragraph_2. This is a sample sentence in paragraph_2. This is a sample sentence in
|
|
|
|
paragraph_2. This is a sample sentence in paragraph_2. This is a sample sentence in paragraph_2.
|
|
|
|
|
|
|
|
This is a sample sentence in paragraph_3. This is a sample sentence in paragraph_3. This is a sample sentence in
|
|
|
|
paragraph_3. This is a sample sentence in paragraph_3. This is to trick the test with using an abbreviation like Dr.
|
|
|
|
in the sentence.
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
def test_preprocess_sentence_split():
|
2022-03-29 13:53:35 +02:00
|
|
|
document = Document(content=TEXT)
|
2022-02-03 13:43:18 +01:00
|
|
|
preprocessor = PreProcessor(
|
|
|
|
split_length=1, split_overlap=0, split_by="sentence", split_respect_sentence_boundary=False
|
|
|
|
)
|
2020-10-15 10:42:08 +02:00
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 15
|
|
|
|
|
|
|
|
preprocessor = PreProcessor(
|
2022-03-07 19:25:33 +01:00
|
|
|
split_length=10, split_overlap=0, split_by="sentence", split_respect_sentence_boundary=False
|
2020-10-15 10:42:08 +02:00
|
|
|
)
|
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 2
|
|
|
|
|
|
|
|
|
|
|
|
def test_preprocess_word_split():
|
2022-03-29 13:53:35 +02:00
|
|
|
document = Document(content=TEXT)
|
2022-02-03 13:43:18 +01:00
|
|
|
preprocessor = PreProcessor(
|
|
|
|
split_length=10, split_overlap=0, split_by="word", split_respect_sentence_boundary=False
|
|
|
|
)
|
2020-10-15 10:42:08 +02:00
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 11
|
|
|
|
|
2021-01-06 15:54:05 +01:00
|
|
|
preprocessor = PreProcessor(split_length=15, split_overlap=0, split_by="word", split_respect_sentence_boundary=True)
|
2020-10-15 10:42:08 +02:00
|
|
|
documents = preprocessor.process(document)
|
2022-02-03 13:43:18 +01:00
|
|
|
for i, doc in enumerate(documents):
|
2020-11-13 14:14:24 +01:00
|
|
|
if i == 0:
|
2022-03-29 13:53:35 +02:00
|
|
|
assert len(doc.content.split(" ")) == 14
|
|
|
|
assert len(doc.content.split(" ")) <= 15 or doc.content.startswith("This is to trick")
|
2020-11-13 14:14:24 +01:00
|
|
|
assert len(documents) == 8
|
2020-10-15 10:42:08 +02:00
|
|
|
|
2022-02-03 13:43:18 +01:00
|
|
|
preprocessor = PreProcessor(
|
|
|
|
split_length=40, split_overlap=10, split_by="word", split_respect_sentence_boundary=True
|
|
|
|
)
|
2020-12-09 16:12:36 +01:00
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 5
|
|
|
|
|
2021-02-01 17:08:27 +01:00
|
|
|
preprocessor = PreProcessor(split_length=5, split_overlap=0, split_by="word", split_respect_sentence_boundary=True)
|
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 15
|
|
|
|
|
2020-10-15 10:42:08 +02:00
|
|
|
|
|
|
|
def test_preprocess_passage_split():
|
2022-03-29 13:53:35 +02:00
|
|
|
document = Document(content=TEXT)
|
2022-02-03 13:43:18 +01:00
|
|
|
preprocessor = PreProcessor(
|
|
|
|
split_length=1, split_overlap=0, split_by="passage", split_respect_sentence_boundary=False
|
|
|
|
)
|
2020-10-15 10:42:08 +02:00
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 3
|
|
|
|
|
2022-02-03 13:43:18 +01:00
|
|
|
preprocessor = PreProcessor(
|
|
|
|
split_length=2, split_overlap=0, split_by="passage", split_respect_sentence_boundary=False
|
|
|
|
)
|
2020-10-15 10:42:08 +02:00
|
|
|
documents = preprocessor.process(document)
|
|
|
|
assert len(documents) == 2
|
|
|
|
|
|
|
|
|
|
|
|
def test_clean_header_footer():
|
|
|
|
converter = PDFToTextConverter()
|
2022-02-03 13:43:18 +01:00
|
|
|
document = converter.convert(
|
|
|
|
file_path=Path(SAMPLES_PATH / "pdf" / "sample_pdf_2.pdf")
|
|
|
|
) # file contains header/footer
|
2020-10-15 10:42:08 +02:00
|
|
|
|
|
|
|
preprocessor = PreProcessor(clean_header_footer=True, split_by=None)
|
|
|
|
documents = preprocessor.process(document)
|
|
|
|
|
|
|
|
assert len(documents) == 1
|
|
|
|
|
2022-03-29 13:53:35 +02:00
|
|
|
assert "This is a header." not in documents[0].content
|
|
|
|
assert "footer" not in documents[0].content
|
2022-03-08 15:49:56 +01:00
|
|
|
|
|
|
|
|
|
|
|
def test_remove_substrings():
|
2022-03-29 13:53:35 +02:00
|
|
|
document = Document(content="This is a header. Some additional text. wiki. Some emoji ✨ 🪲 Weird whitespace\b\b\b.")
|
2022-03-08 15:49:56 +01:00
|
|
|
|
|
|
|
# check that the file contains the substrings we are about to remove
|
2022-03-29 13:53:35 +02:00
|
|
|
assert "This is a header." in document.content
|
|
|
|
assert "wiki" in document.content
|
|
|
|
assert "🪲" in document.content
|
|
|
|
assert "whitespace" in document.content
|
|
|
|
assert "✨" in document.content
|
2022-03-08 15:49:56 +01:00
|
|
|
|
|
|
|
preprocessor = PreProcessor(remove_substrings=["This is a header.", "wiki", "🪲"])
|
|
|
|
documents = preprocessor.process(document)
|
|
|
|
|
2022-03-29 13:53:35 +02:00
|
|
|
assert "This is a header." not in documents[0].content
|
|
|
|
assert "wiki" not in documents[0].content
|
|
|
|
assert "🪲" not in documents[0].content
|
|
|
|
assert "whitespace" in documents[0].content
|
|
|
|
assert "✨" in documents[0].content
|