Fix skipping of tests using document stores (#2268)

* Fix skipping document store tests

* Update Documentation & Code Style

* Fix handling of Milvus1 and Milvus2 in tests

* Update Documentation & Code Style

* Fix handling of Milvus1 and Milvus2 in tests

* Update Documentation & Code Style

* Remove SQL from tests requiring embeddings

* Update Documentation & Code Style

* Fix get_embedding_count of Milvus2

* Make sure to start Milvus2 tests with a new collection

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

conftest.py

@@ -1,5 +1,7 @@
 def pytest_addoption(parser):
-    parser.addoption("--document_store_type", action="store", default="elasticsearch, faiss, memory, milvus, weaviate")
+    parser.addoption(
+        "--document_store_type", action="store", default="elasticsearch, faiss, sql, memory, milvus1, milvus, weaviate"
+    )
 
 
 def pytest_generate_tests(metafunc):

haystack/document_stores/milvus2.py

@@ -661,4 +661,4 @@ class Milvus2DocumentStore(SQLDocumentStore):
         """
         if filters:
             raise Exception("filters are not supported for get_embedding_count in MilvusDocumentStore.")
-        return len(self.get_all_documents())
+        return len(self.get_all_documents(index=index))

test/conftest.py

@@ -116,17 +116,9 @@ def pytest_collection_modifyitems(config, items):
         # if the cli argument "--document_store_type" is used, we want to skip all tests that have markers of other docstores
         # Example: pytest -v test_document_store.py --document_store_type="memory" => skip all tests marked with "elasticsearch"
         document_store_types_to_run = config.getoption("--document_store_type")
-        document_store_types_to_run = document_store_types_to_run.split(",")
+        document_store_types_to_run = document_store_types_to_run.split(", ")
         keywords = []
 
-        if "milvus1" in document_store_types_to_run and not os.getenv("MILVUS1_ENABLED"):
-            document_store_types_to_run.remove("milvus1")
-            document_store_types_to_run.append("milvus")
-            if not milvus1:
-                raise Exception(
-                    "Milvus1 is enabled, but your pymilvus version only supports Milvus 2. Please select the correct pymilvus version."
-                )
-
         for i in item.keywords:
             if "-" in i:
                 keywords.extend(i.split("-"))
@@ -139,6 +131,13 @@ def pytest_collection_modifyitems(config, items):
                 )
                 item.add_marker(skip_docstore)
 
+        if "milvus1" in keywords and not milvus1:
+            skip_milvus1 = pytest.mark.skip(reason="Skipping Tests for 'milvus1', as Milvus2 seems to be installed.")
+            item.add_marker(skip_milvus1)
+        elif "milvus" in keywords and milvus1:
+            skip_milvus = pytest.mark.skip(reason="Skipping Tests for 'milvus', as Milvus1 seems to be installed.")
+            item.add_marker(skip_milvus)
+
 
 @pytest.fixture(scope="function", autouse=True)
 def gc_cleanup(request):
@@ -563,6 +562,10 @@ def document_store(request, tmp_path):
     yield document_store
     document_store.delete_documents()
 
+    # Make sure to drop Milvus2 collection, required for tests using different embedding dimensions
+    if isinstance(document_store, MilvusDocumentStore) and not milvus1:
+        document_store.collection.drop()
+
 
 @pytest.fixture(params=["memory", "faiss", "milvus1", "milvus", "elasticsearch"])
 def document_store_dot_product(request, tmp_path):

test/test_document_store.py

@@ -450,6 +450,9 @@ def test_write_document_index(document_store):
     assert len(document_store.get_all_documents()) == 0
 
 
+@pytest.mark.parametrize(
+    "document_store", ["elasticsearch", "faiss", "memory", "milvus1", "milvus", "weaviate"], indirect=True
+)
 def test_document_with_embeddings(document_store):
     documents = [
         {"content": "text1", "id": "1", "embedding": np.random.rand(768).astype(np.float32)},
@@ -471,6 +474,9 @@ def test_document_with_embeddings(document_store):
     assert isinstance(documents_with_embedding[0].embedding, (list, np.ndarray))
 
 
+@pytest.mark.parametrize(
+    "document_store", ["elasticsearch", "faiss", "memory", "milvus1", "milvus", "weaviate"], indirect=True
+)
 @pytest.mark.parametrize("retriever", ["embedding"], indirect=True)
 def test_update_embeddings(document_store, retriever):
     documents = []
@@ -582,6 +588,7 @@ def test_update_embeddings(document_store, retriever):
         assert document_store.get_embedding_count(index="haystack_test_one") == 14
 
 
+@pytest.mark.parametrize("document_store", ["elasticsearch"], indirect=True)
 @pytest.mark.parametrize("retriever", ["table_text_retriever"], indirect=True)
 @pytest.mark.embedding_dim(512)
 def test_update_embeddings_table_text_retriever(document_store, retriever):

test/test_retriever.py

@@ -150,6 +150,9 @@ def test_elasticsearch_custom_query():
 
 
 @pytest.mark.slow
+@pytest.mark.parametrize(
+    "document_store", ["elasticsearch", "faiss", "memory", "milvus1", "milvus", "weaviate"], indirect=True
+)
 @pytest.mark.parametrize("retriever", ["dpr"], indirect=True)
 def test_dpr_embedding(document_store, retriever, docs):
 
@@ -178,6 +181,9 @@ def test_dpr_embedding(document_store, retriever, docs):
 
 
 @pytest.mark.slow
+@pytest.mark.parametrize(
+    "document_store", ["elasticsearch", "faiss", "memory", "milvus1", "milvus", "weaviate"], indirect=True
+)
 @pytest.mark.parametrize("retriever", ["retribert"], indirect=True)
 @pytest.mark.embedding_dim(128)
 def test_retribert_embedding(document_store, retriever, docs):

test/test_standard_pipelines.py

@@ -77,6 +77,9 @@ def test_faq_pipeline(retriever, document_store):
 
 
 @pytest.mark.parametrize("retriever", ["embedding"], indirect=True)
+@pytest.mark.parametrize(
+    "document_store", ["elasticsearch", "faiss", "memory", "milvus1", "milvus", "weaviate"], indirect=True
+)
 def test_document_search_pipeline(retriever, document_store):
     documents = [
         {"content": "Sample text for document-1", "meta": {"source": "wiki1"}},