mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-08-20 06:28:39 +00:00
d470b9d0bd
* Fist attempt at using setup.cfg for dependency management * Trying the new package on the CI and in Docker too * Add composite extras_require * Add the safe_import function for document store imports and add some try-catch statements on rest_api and ui imports * Fix bug on class import and rephrase error message * Introduce typing for optional modules and add type: ignore in sparse.py * Include importlib_metadata backport for py3.7 * Add colab group to extra_requires * Fix pillow version * Fix grpcio * Separate out the crawler as another extra * Make paths relative in rest_api and ui * Update the test matrix in the CI * Add try catch statements around the optional imports too to account for direct imports * Never mix direct deps with self-references and add ES deps to the base install * Refactor several paths in tests to make them insensitive to the execution path * Include tstadel review and re-introduce Milvus1 in the tests suite, to fix * Wrap pdf conversion utils into safe_import * Update some tutorials and rever Milvus1 as default for now, see #2067 * Fix mypy config Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
109 lines
4.3 KiB
Python
109 lines
4.3 KiB
Python
from pathlib import Path
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from haystack.nodes import MarkdownConverter, DocxToTextConverter, PDFToTextConverter, PDFToTextOCRConverter, \
|
|
TikaConverter, AzureConverter, ParsrConverter
|
|
|
|
from conftest import SAMPLES_PATH
|
|
|
|
|
|
|
|
@pytest.mark.tika
|
|
@pytest.mark.parametrize(
|
|
# "Converter", [PDFToTextConverter, TikaConverter, PDFToTextOCRConverter]
|
|
"Converter", [PDFToTextOCRConverter]
|
|
)
|
|
def test_convert(Converter):
|
|
converter = Converter()
|
|
document = converter.convert(file_path=SAMPLES_PATH/"pdf"/"sample_pdf_1.pdf")[0]
|
|
pages = document["content"].split("\f")
|
|
assert len(pages) == 4 # the sample PDF file has four pages.
|
|
assert pages[0] != "" # the page 1 of PDF contains text.
|
|
assert pages[2] == "" # the page 3 of PDF file is empty.
|
|
# assert text is retained from the document.
|
|
# As whitespace can differ (\n," ", etc.), we standardize all to simple whitespace
|
|
page_standard_whitespace = " ".join(pages[0].split())
|
|
assert (
|
|
"Adobe Systems made the PDF specification available free of charge in 1993."
|
|
in page_standard_whitespace
|
|
)
|
|
|
|
|
|
@pytest.mark.tika
|
|
@pytest.mark.parametrize("Converter", [PDFToTextConverter, TikaConverter])
|
|
def test_table_removal(Converter):
|
|
converter = Converter(remove_numeric_tables=True)
|
|
document = converter.convert(file_path=SAMPLES_PATH/"pdf"/"sample_pdf_1.pdf")[0]
|
|
pages = document["content"].split("\f")
|
|
# assert numeric rows are removed from the table.
|
|
assert "324" not in pages[0]
|
|
assert "54x growth" not in pages[0]
|
|
|
|
|
|
@pytest.mark.tika
|
|
@pytest.mark.parametrize("Converter", [PDFToTextConverter, TikaConverter])
|
|
def test_language_validation(Converter, caplog):
|
|
converter = Converter(valid_languages=["en"])
|
|
converter.convert(file_path=SAMPLES_PATH/"pdf"/"sample_pdf_1.pdf")
|
|
assert (
|
|
"samples/pdf/sample_pdf_1.pdf is not one of ['en']."
|
|
not in caplog.text
|
|
)
|
|
|
|
converter = Converter(valid_languages=["de"])
|
|
converter.convert(file_path=SAMPLES_PATH/"pdf"/"sample_pdf_1.pdf")
|
|
assert (
|
|
"samples/pdf/sample_pdf_1.pdf is not one of ['de']."
|
|
in caplog.text
|
|
)
|
|
|
|
|
|
def test_docx_converter():
|
|
converter = DocxToTextConverter()
|
|
document = converter.convert(file_path=SAMPLES_PATH/"docx"/"sample_docx.docx")[0]
|
|
assert document["content"].startswith("Sample Docx File")
|
|
|
|
|
|
def test_markdown_converter():
|
|
converter = MarkdownConverter()
|
|
document = converter.convert(file_path=SAMPLES_PATH/"markdown"/"sample.md")[0]
|
|
assert document["content"].startswith("What to build with Haystack")
|
|
|
|
|
|
def test_azure_converter():
|
|
# Check if Form Recognizer endpoint and credential key in environment variables
|
|
if "AZURE_FORMRECOGNIZER_ENDPOINT" in os.environ and "AZURE_FORMRECOGNIZER_KEY" in os.environ:
|
|
converter = AzureConverter(endpoint=os.environ["AZURE_FORMRECOGNIZER_ENDPOINT"],
|
|
credential_key=os.environ["AZURE_FORMRECOGNIZER_KEY"],
|
|
save_json=True,
|
|
)
|
|
|
|
docs = converter.convert(file_path=SAMPLES_PATH/"pdf"/"sample_pdf_1.pdf")
|
|
assert len(docs) == 2
|
|
assert docs[0]["content_type"] == "table"
|
|
assert len(docs[0]["content"]) == 5 # number of rows
|
|
assert len(docs[0]["content"][0]) == 5 # number of columns, Form Recognizer assumes there are 5 columns
|
|
assert docs[0]["content"][0] == ['', 'Column 1', '', 'Column 2', 'Column 3']
|
|
assert docs[0]["content"][4] == ['D', '$54.35', '', '$6345.', '']
|
|
|
|
assert docs[1]["content_type"] == "text"
|
|
assert docs[1]["content"].startswith("A sample PDF file")
|
|
|
|
|
|
def test_parsr_converter():
|
|
converter = ParsrConverter()
|
|
|
|
docs = converter.convert(file_path=str((SAMPLES_PATH/"pdf"/"sample_pdf_1.pdf").absolute()))
|
|
assert len(docs) == 2
|
|
assert docs[0]["content_type"] == "table"
|
|
assert len(docs[0]["content"]) == 5 # number of rows
|
|
assert len(docs[0]["content"][0]) == 4 # number of columns
|
|
assert docs[0]["content"][0] == ['', 'Column 1', 'Column 2', 'Column 3']
|
|
assert docs[0]["content"][4] == ['D', '$54.35', '$6345.', '']
|
|
|
|
assert docs[1]["content_type"] == "text"
|
|
assert docs[1]["content"].startswith("A sample PDF file")
|
|
assert docs[1]["content"].endswith("Page 4 of Sample PDF\n… the page 3 is empty.")
|