fix: overwrite params with environment variables even if there are no params in the pipeline definition; make mypy ignore REST API tests (#3930)

* fix and new test

* make mypy ignore rest_api tests files

* try to improve mypy action

* retry

* fix

* test new action

* ok

* check python files not in root

* really check files!

.github/workflows/tests.yml

@@ -89,7 +89,7 @@ jobs:
       uses: tj-actions/changed-files@v34
       with:
         files: |
-          *.py
+          **/*.py
         files_ignore: |
           test/**
           rest_api/test/**
@@ -100,10 +100,10 @@ jobs:
         pythonVersion: 3.8   # Literals and mypy just don't work well together.
 
     - name: Mypy
-      if: steps.python-files.outputs.changed_files
+      if: steps.files.outputs.any_changed == 'true'
       run: |
         mkdir .mypy_cache/
-        mypy --install-types --non-interactive ${{ steps.python-files.outputs.changed_files }} --exclude=rest_api/build/ --exclude=rest_api/test/
+        mypy --install-types --non-interactive ${{ steps.files.outputs.all_changed_files }} --exclude=rest_api/build/ --exclude=rest_api/test/
 
     - uses: act10ns/slack@v1
       with:

haystack/pipelines/config.py

@@ -79,6 +79,8 @@ def get_component_definitions(
                 env_prefix = f"{name}_params_".upper()
                 if key.startswith(env_prefix):
                     param_name = key.replace(env_prefix, "").lower()
+                    if "params" not in component_definition:
+                        component_definition["params"] = {}
                     component_definition["params"][param_name] = value
                     logger.info(
                         "Param '%s' of component '%s' overwritten with environment variable '%s' value '%s'.",

rest_api/test/samples/test.docstore-no-params-pipeline.yml

@@ -0,0 +1,15 @@
+version: 'ignore'
+
+components:
+  - name: InMemoryDocumentStore
+    type: InMemoryDocumentStore
+  - name: tfidf_ret
+    params:
+      document_store: InMemoryDocumentStore
+    type: TfidfRetriever
+pipelines:
+  - name: query
+    nodes:
+      - inputs:
+          - Query
+        name: tfidf_ret

rest_api/test/test_rest_api.py

@@ -40,6 +40,13 @@ def test_check_error_for_pipeline_not_found():
     assert p is None
 
 
+def test_overwrite_params_with_env_variables_when_no_params_in_pipeline_yaml(monkeypatch):
+    yaml_pipeline_path = Path(__file__).parent.resolve() / "samples" / "test.docstore-no-params-pipeline.yml"
+    monkeypatch.setenv("INMEMORYDOCUMENTSTORE_PARAMS_INDEX", "custom_index")
+    _, document_store = _load_pipeline(yaml_pipeline_path, None)
+    assert document_store.index == "custom_index"
+
+
 def test_bad_yaml_pipeline_configuration_error():
 
     yaml_pipeline_path = Path(__file__).parent.resolve() / "samples" / "test.bogus_pipeline.yml"