Fix windows ci tests (#2144)

* move commandline args to global conftest

* correct test exclude paths

* Update Documentation & Code Style

* exclude test_generator_pipeline_with_translator from windows ci

* exclude further oom tests

* enable log_cli

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/windows_ci.yml

@@ -101,5 +101,5 @@ jobs:
     # As on windows we are going to disable quite a few tests these, hence these files will throw error refer https://github.com/pytest-dev/pytest/issues/812
     # Removing test_ray, test_utils, test_preprocessor, test_knowledge_graph and test_connector
     - name: Run tests
-      if: ${{ !contains(fromJSON('["test_ray.py", "test_knowledge_graph.py", "test_connector.py", "test_summarizer_translation.py"]'), matrix.test-path) }}
+      if: ${{ !contains(fromJSON('["./test/test_ray.py", "./test/test_knowledge_graph.py", "./test/test_connector.py", "./test/test_summarizer_translation.py", "./test/test_summarizer.py"]'), matrix.test-path) }}
       run: pytest --document_store_type=memory,faiss,elasticsearch -m "not tika and not graphdb" -k "not test_parsr_converter" -s ${{ matrix.test-path }}

conftest.py

@@ -0,0 +1,20 @@
+def pytest_addoption(parser):
+    parser.addoption("--document_store_type", action="store", default="elasticsearch, faiss, memory, milvus, weaviate")
+
+
+def pytest_generate_tests(metafunc):
+    # Get selected docstores from CLI arg
+    document_store_type = metafunc.config.option.document_store_type
+    selected_doc_stores = [item.strip() for item in document_store_type.split(",")]
+
+    # parametrize document_store fixture if it's in the test function argument list
+    # but does not have an explicit parametrize annotation e.g
+    # @pytest.mark.parametrize("document_store", ["memory"], indirect=False)
+    found_mark_parametrize_document_store = False
+    for marker in metafunc.definition.iter_markers("parametrize"):
+        if "document_store" in marker.args[0]:
+            found_mark_parametrize_document_store = True
+            break
+    # for all others that don't have explicit parametrization, we add the ones from the CLI arg
+    if "document_store" in metafunc.fixturenames and not found_mark_parametrize_document_store:
+        metafunc.parametrize("document_store", selected_doc_stores, indirect=True)

pyproject.toml

@@ -127,4 +127,5 @@ markers = [
     "summarizer: marks summarizer tests",
     "weaviate: marks tests that require weaviate container",
     "embedding_dim: marks usage of document store with non-default embedding dimension (e.g @pytest.mark.embedding_dim(128))",
-]
+]
+log_cli = true

test/conftest.py

@@ -64,28 +64,6 @@ DC_API_KEY = "NO_KEY"
 MOCK_DC = True
 
 
-def pytest_addoption(parser):
-    parser.addoption("--document_store_type", action="store", default="elasticsearch, faiss, memory, milvus, weaviate")
-
-
-def pytest_generate_tests(metafunc):
-    # Get selected docstores from CLI arg
-    document_store_type = metafunc.config.option.document_store_type
-    selected_doc_stores = [item.strip() for item in document_store_type.split(",")]
-
-    # parametrize document_store fixture if it's in the test function argument list
-    # but does not have an explicit parametrize annotation e.g
-    # @pytest.mark.parametrize("document_store", ["memory"], indirect=False)
-    found_mark_parametrize_document_store = False
-    for marker in metafunc.definition.iter_markers("parametrize"):
-        if "document_store" in marker.args[0]:
-            found_mark_parametrize_document_store = True
-            break
-    # for all others that don't have explicit parametrization, we add the ones from the CLI arg
-    if "document_store" in metafunc.fixturenames and not found_mark_parametrize_document_store:
-        metafunc.parametrize("document_store", selected_doc_stores, indirect=True)
-
-
 def _sql_session_rollback(self, attr):
     """
     Inject SQLDocumentStore at runtime to do a session rollback each time it is called. This allows to catch

test/test_eval.py

@@ -1,4 +1,5 @@
 import pytest
+import sys
 from pathlib import Path
 from haystack.document_stores.base import BaseDocumentStore
 from haystack.document_stores.memory import InMemoryDocumentStore
@@ -24,6 +25,7 @@ from haystack.schema import Answer, Document, EvaluationResult, Label, MultiLabe
 from conftest import SAMPLES_PATH
 
 
+@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner")
 @pytest.mark.parametrize("document_store_with_docs", ["memory"], indirect=True)
 @pytest.mark.parametrize("retriever_with_docs", ["embedding"], indirect=True)
 def test_generativeqa_calculate_metrics(
@@ -49,6 +51,7 @@ def test_generativeqa_calculate_metrics(
     assert metrics["Generator"]["f1"] == 1.0 / 3
 
 
+@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner")
 @pytest.mark.parametrize("document_store_with_docs", ["memory"], indirect=True)
 @pytest.mark.parametrize("retriever_with_docs", ["embedding"], indirect=True)
 def test_summarizer_calculate_metrics(

test/test_generator.py

@@ -13,6 +13,7 @@ from conftest import DOCS_WITH_EMBEDDINGS
 
 
 # Keeping few (retriever,document_store) combination to reduce test time
+@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner")
 @pytest.mark.slow
 @pytest.mark.generator
 @pytest.mark.parametrize(
@@ -59,7 +60,7 @@ def test_generator_pipeline(document_store, retriever, rag_generator):
     assert "berlin" in answers[0].answer
 
 
-@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Gives memory allocation error on windows runner")
+@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner")
 @pytest.mark.slow
 @pytest.mark.generator
 @pytest.mark.parametrize("document_store", ["memory"], indirect=True)