From 281f9ff9709342bebd0f5b0e47bbab5a266b89e1 Mon Sep 17 00:00:00 2001 From: Tanay Soni Date: Mon, 11 Jan 2021 13:24:38 +0100 Subject: [PATCH] Fix SQLite errors in tests (#723) --- test/conftest.py | 9 ++------- test/test_faiss.py | 42 ++++++++++++++++++------------------------ 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 83707143a..ffc5689b1 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,3 @@ -import os import subprocess import time from subprocess import run @@ -247,9 +246,7 @@ def document_store(request, test_docs_xs): def get_document_store(document_store_type, embedding_field="embedding"): if document_store_type == "sql": - if os.path.exists("haystack_test.db"): - os.remove("haystack_test.db") - document_store = SQLDocumentStore(url="sqlite:///haystack_test.db") + document_store = SQLDocumentStore(url="sqlite://") elif document_store_type == "memory": document_store = InMemoryDocumentStore(return_embedding=True, embedding_field=embedding_field) elif document_store_type == "elasticsearch": @@ -260,10 +257,8 @@ def get_document_store(document_store_type, embedding_field="embedding"): index="haystack_test", return_embedding=True, embedding_field=embedding_field ) elif document_store_type == "faiss": - if os.path.exists("haystack_test_faiss.db"): - os.remove("haystack_test_faiss.db") document_store = FAISSDocumentStore( - sql_url="sqlite:///haystack_test_faiss.db", return_embedding=True, embedding_field=embedding_field + sql_url="sqlite://", return_embedding=True, embedding_field=embedding_field ) return document_store else: diff --git a/test/test_faiss.py b/test/test_faiss.py index 52042f2a9..0000d42d0 100644 --- a/test/test_faiss.py +++ b/test/test_faiss.py @@ -33,31 +33,25 @@ def check_data_correctness(documents_indexed, documents_inserted): vector_ids.add(doc.meta["vector_id"]) assert len(vector_ids) == len(documents_inserted) -#TODO Test is failing in the CI all of sudden while running smoothly locally. Fix it in a separate PR -# (sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) disk I/O error) -# @pytest.mark.parametrize("document_store", ["faiss"], indirect=True) -# def test_faiss_index_save_and_load(document_store): -# import os -# files = os.listdir(os.curdir) -# print(f"Files in Directory: {files}") -# document_store.write_documents(DOCUMENTS) -# -# # test saving the index -# document_store.save("haystack_test_faiss") -# -# # clear existing faiss_index -# document_store.faiss_index.reset() -# -# # test faiss index is cleared -# assert document_store.faiss_index.ntotal == 0 -# -# # test loading the index -# new_document_store = document_store.load(sql_url="sqlite:///haystack_test.db", -# faiss_file_path="haystack_test_faiss") -# -# # check faiss index is restored -# assert new_document_store.faiss_index.ntotal == len(DOCUMENTS) +@pytest.mark.parametrize("document_store", ["faiss"], indirect=True) +def test_faiss_index_save_and_load(document_store): + document_store.write_documents(DOCUMENTS) + + # test saving the index + document_store.save("haystack_test_faiss") + + # clear existing faiss_index + document_store.faiss_index.reset() + + # test faiss index is cleared + assert document_store.faiss_index.ntotal == 0 + + # test loading the index + new_document_store = FAISSDocumentStore.load(sql_url="sqlite://", faiss_file_path="haystack_test_faiss") + + # check faiss index is restored + assert new_document_store.faiss_index.ntotal == len(DOCUMENTS) @pytest.mark.parametrize("document_store", ["faiss"], indirect=True)