yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-12-12 15:27:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix SQLite errors in tests (#723)

Browse Source
This commit is contained in:
Tanay Soni 2021-01-11 13:24:38 +01:00 committed by GitHub
parent fcc052b554
commit 281f9ff970
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
test/conftest.py
View File

@ -1,4 +1,3 @@
import os
import subprocess
import time
from subprocess import run
@ -247,9 +246,7 @@ def document_store(request, test_docs_xs):
def get_document_store(document_store_type, embedding_field="embedding"):
if document_store_type == "sql":
if os.path.exists("haystack_test.db"):
os.remove("haystack_test.db")
document_store = SQLDocumentStore(url="sqlite:///haystack_test.db")
document_store = SQLDocumentStore(url="sqlite://")
elif document_store_type == "memory":
document_store = InMemoryDocumentStore(return_embedding=True, embedding_field=embedding_field)
elif document_store_type == "elasticsearch":
@ -260,10 +257,8 @@ def get_document_store(document_store_type, embedding_field="embedding"):
index="haystack_test", return_embedding=True, embedding_field=embedding_field
)
elif document_store_type == "faiss":
if os.path.exists("haystack_test_faiss.db"):
os.remove("haystack_test_faiss.db")
document_store = FAISSDocumentStore(
sql_url="sqlite:///haystack_test_faiss.db", return_embedding=True, embedding_field=embedding_field
sql_url="sqlite://", return_embedding=True, embedding_field=embedding_field
)
return document_store
else:

42
test/test_faiss.py
View File

@ -33,31 +33,25 @@ def check_data_correctness(documents_indexed, documents_inserted):
vector_ids.add(doc.meta["vector_id"])
assert len(vector_ids) == len(documents_inserted)
#TODO Test is failing in the CI all of sudden while running smoothly locally. Fix it in a separate PR
# (sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) disk I/O error)
# @pytest.mark.parametrize("document_store", ["faiss"], indirect=True)
# def test_faiss_index_save_and_load(document_store):
# import os
# files = os.listdir(os.curdir)
# print(f"Files in Directory: {files}")
# document_store.write_documents(DOCUMENTS)
#
# # test saving the index
# document_store.save("haystack_test_faiss")
#
# # clear existing faiss_index
# document_store.faiss_index.reset()
#
# # test faiss index is cleared
# assert document_store.faiss_index.ntotal == 0
#
# # test loading the index
# new_document_store = document_store.load(sql_url="sqlite:///haystack_test.db",
# faiss_file_path="haystack_test_faiss")
#
# # check faiss index is restored
# assert new_document_store.faiss_index.ntotal == len(DOCUMENTS)
@pytest.mark.parametrize("document_store", ["faiss"], indirect=True)
def test_faiss_index_save_and_load(document_store):
document_store.write_documents(DOCUMENTS)
# test saving the index
document_store.save("haystack_test_faiss")
# clear existing faiss_index
document_store.faiss_index.reset()
# test faiss index is cleared
assert document_store.faiss_index.ntotal == 0
# test loading the index
new_document_store = FAISSDocumentStore.load(sql_url="sqlite://", faiss_file_path="haystack_test_faiss")
# check faiss index is restored
assert new_document_store.faiss_index.ntotal == len(DOCUMENTS)
@pytest.mark.parametrize("document_store", ["faiss"], indirect=True)