mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-28 19:29:40 +00:00
e5b4b62d75
* Feat: Removing use of temp file while downloading archive from url along with adding CI for windows and mac platform * Windows CI by default installing pytorch gpu hence updating CI to pick cpu version * fixing mac cache build issue * updating windows pip install command for torch * another attempt * updating ci * Adding sudo * fixing ls failure on windows * another attempt to fix build issue * Saving env variable of test files * Adding debug log * Github action differ on windows * adding debug * anohter attempt * Windows have different ways to receive env * fixing template * minor fx * Adding debug * Removing use of json * Adding back fromJson * addin toJson * removing print * anohter attempt * disabling parallel run at least for testing * installing docker for mac runner * correcting docker install command * Linux dockers are not suported in windows * Removing mac changes * Upgrading pytorch * using lts pytorch * Separating win and ubuntu * Install java 11 * enabling linux container env * docker cli command * docker cli command * start elastic service * List all service * correcting service name * Attempt to fix multiple test run * convert to json * another attempt to check * Updating build cache step * attempt * Add tika * Separating windows CI * Changing CI name * Skipping test which does not work in windows * Skipping tests for windows * create cleanup function in conftest * adding skipif marker on tests * Run windows PR on only push to master * Addressing review comments * Enabling windows ci for this PR * Tika init is being called when importing tika function * handling tika import issue * handling tika import issue in test * Fixing import issue * removing tika fixure * Removing fixture from tests * Disable windows ci on pull request * Add back extra pytorch install step Co-authored-by: Malte Pietsch <malte.pietsch@deepset.ai>
71 lines
2.6 KiB
Python
71 lines
2.6 KiB
Python
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from haystack.file_converter import MarkdownConverter
|
|
from haystack.file_converter.docx import DocxToTextConverter
|
|
from haystack.file_converter.pdf import PDFToTextConverter, PDFToTextOCRConverter
|
|
from haystack.file_converter.tika import TikaConverter
|
|
|
|
|
|
@pytest.mark.tika
|
|
@pytest.mark.parametrize(
|
|
# "Converter", [PDFToTextConverter, TikaConverter, PDFToTextOCRConverter]
|
|
"Converter", [PDFToTextOCRConverter]
|
|
)
|
|
def test_convert(Converter):
|
|
converter = Converter()
|
|
document = converter.convert(file_path=Path("samples/pdf/sample_pdf_1.pdf"))
|
|
pages = document["content"].split("\f")
|
|
assert len(pages) == 4 # the sample PDF file has four pages.
|
|
assert pages[0] != "" # the page 1 of PDF contains text.
|
|
assert pages[2] == "" # the page 3 of PDF file is empty.
|
|
# assert text is retained from the document.
|
|
# As whitespace can differ (\n," ", etc.), we standardize all to simple whitespace
|
|
page_standard_whitespace = " ".join(pages[0].split())
|
|
assert (
|
|
"Adobe Systems made the PDF specification available free of charge in 1993."
|
|
in page_standard_whitespace
|
|
)
|
|
|
|
|
|
@pytest.mark.tika
|
|
@pytest.mark.parametrize("Converter", [PDFToTextConverter, TikaConverter])
|
|
def test_table_removal(Converter):
|
|
converter = Converter(remove_numeric_tables=True)
|
|
document = converter.convert(file_path=Path("samples/pdf/sample_pdf_1.pdf"))
|
|
pages = document["content"].split("\f")
|
|
# assert numeric rows are removed from the table.
|
|
assert "324" not in pages[0]
|
|
assert "54x growth" not in pages[0]
|
|
|
|
|
|
@pytest.mark.tika
|
|
@pytest.mark.parametrize("Converter", [PDFToTextConverter, TikaConverter])
|
|
def test_language_validation(Converter, caplog):
|
|
converter = Converter(valid_languages=["en"])
|
|
converter.convert(file_path=Path("samples/pdf/sample_pdf_1.pdf"))
|
|
assert (
|
|
"The language for samples/pdf/sample_pdf_1.pdf is not one of ['en']."
|
|
not in caplog.text
|
|
)
|
|
|
|
converter = Converter(valid_languages=["de"])
|
|
converter.convert(file_path=Path("samples/pdf/sample_pdf_1.pdf"))
|
|
assert (
|
|
"The language for samples/pdf/sample_pdf_1.pdf is not one of ['de']."
|
|
in caplog.text
|
|
)
|
|
|
|
|
|
def test_docx_converter():
|
|
converter = DocxToTextConverter()
|
|
document = converter.convert(file_path=Path("samples/docx/sample_docx.docx"))
|
|
assert document["content"].startswith("Sample Docx File")
|
|
|
|
|
|
def test_markdown_converter():
|
|
converter = MarkdownConverter()
|
|
document = converter.convert(file_path=Path("samples/markdown/sample.md"))
|
|
assert document["content"].startswith("What to build with Haystack")
|