ci: remove python_cache internal action (#4429)

.github/actions/python_cache/action.yml

@@ -1,51 +0,0 @@
-name: Python Cache
-description: Caches a full installation of Haystack dependencies
-
-inputs:
-  prefix:
-    description: 'prefix for the cache (to distinguish OS)'
-    required: true
-    default: 'linux'
-  pythonVersion:
-    description: 'Python version to use'
-    required: true
-    default: "3.8"
-
-runs:
-  using: "composite"
-  steps:
-
-  - name: Get date
-    shell: bash
-    run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
-
-  - name: Checkout
-    uses: actions/checkout@v3
-    with:
-      repository: ${{ github.event.pull_request.head.repo.full_name }}
-      ref: ${{ github.head_ref }}
-
-  - name: Setup Python
-    uses: actions/setup-python@v4
-    with:
-      python-version: ${{ inputs.pythonVersion }}
-
-  - name: Cache Python path
-    id: cache-python-env
-    uses: actions/cache@v3
-    with:
-      path: ${{ env.pythonLocation }}
-      # The cache will be rebuild every day and at every change of the dependency files
-      key: ${{ inputs.prefix }}-py${{ inputs.pythonVersion }}-${{ env.date }}-${{ hashFiles('**/pyproject.toml') }}
-
-  - name: Install dependencies
-    # NOTE: this action runs only on cache miss to refresh the dependencies and make sure the next Haystack install will be faster.
-    # Do not rely on this step to install Haystack itself!
-    # The cache can last way longer than a specific action's run, so older Haystack version could be carried over and the tests will run
-    # against the cached code instead of the new changes.
-    if: steps.cache-python-env.outputs.cache-hit != 'true'
-    shell: bash
-    run: |
-      python -m pip install --upgrade pip
-      pip install .[all,alfalfa]
-      pip install rest_api/

.github/workflows/e2e.yml

@@ -4,6 +4,8 @@ name: end-to-end
 on:
   workflow_dispatch:
 
+env:
+  PYTHON_VERSION: "3.8"
 
 jobs:
   e2e:
@@ -19,8 +21,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Cache HF models
       id: cache-hf-models

.github/workflows/examples-tests.yml

@@ -16,6 +16,7 @@ on:
 
 env:
   SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+  PYTHON_VERSION: "3.8"
 
 jobs:
   tests:
@@ -33,8 +34,9 @@ jobs:
       steps:
         - uses: actions/checkout@v3
 
-        - name: Setup Python
-          uses: ./.github/actions/python_cache/
+        - uses: actions/setup-python@v4
+          with:
+            python-version: ${{ env.PYTHON_VERSION }}
 
         - name: Install Haystack
           run: pip install .[all]

.github/workflows/linting.yml

@@ -7,6 +7,9 @@ on:
       - "**.py"
       - "**/pyproject.toml"
 
+env:
+  PYTHON_VERSION: "3.8"
+
 jobs:
   mypy:
     runs-on: ubuntu-latest
@@ -28,8 +31,12 @@ jobs:
             test/**
             rest_api/test/**
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Haystack
+        run: pip install ".[all]"
 
       - name: Mypy
         if: steps.files.outputs.any_changed == 'true'
@@ -57,8 +64,12 @@ jobs:
             test/**
             rest_api/test/**
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install Haystack
+        run: pip install ".[all]"
 
       - name: Pylint
         if: steps.files.outputs.any_changed == 'true'

.github/workflows/minor_version_release.yml

@@ -3,6 +3,9 @@ name: Minor Version Release
 on:
   workflow_dispatch:
 
+env:
+  PYTHON_VERSION: "3.8"
+
 jobs:
   sync:
     runs-on: ubuntu-latest
@@ -10,9 +13,6 @@ jobs:
       - name: Checkout this repo
         uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
-
       - name: Define all versions
         id: versions
         shell: bash
@@ -42,6 +42,13 @@ jobs:
           git push -u origin bump-version
           gh pr create -B main -H bump-version --title 'Bump unstable version' --body 'Part of the release process' --label 'ignore-for-release-notes'
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Install release_docs.py dependencies
+        run: pip install requests
+
       # Note that patch versions all sync to the one readme minor version
       # e.g. Haystack 1.9.1 and 1.9.2 both map to Readme 1.9
       - name: Release Readme version

.github/workflows/openapi_sync.yml

@@ -7,14 +7,18 @@ on:
       # release branches have the form v1.9.x
       - "v[0-9].*[0-9].x"
 
+env:
+  PYTHON_VERSION: "3.8"
+
 jobs:
   run:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install dependencies
         run: sudo apt update && sudo apt-get install libsndfile1 ffmpeg

.github/workflows/rest_api_tests.yml

@@ -17,6 +17,9 @@ on:
       - "rest_api/**.py"
       - "rest_api/pyproject.toml"
 
+env:
+  PYTHON_VERSION: "3.8"
+
 jobs:
   black:
     runs-on: ubuntu-latest
@@ -24,6 +27,8 @@ jobs:
       - uses: actions/checkout@v3
 
       - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Black
         run: |
@@ -73,12 +78,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install REST API
         run: |
           pip install -U "./rest_api[dev]"
+          pip install ".[dev]"
           pip install .
 
       - name: Run tests

.github/workflows/tests.yml

@@ -32,6 +32,7 @@ env:
     --ignore=test/nodes/test_summarizer.py
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+  PYTHON_VERSION: "3.8"
 
 jobs:
 
@@ -42,6 +43,8 @@ jobs:
     - uses: actions/checkout@v3
 
     - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Install Black
       run: |
@@ -99,11 +102,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install .[all,alfalfa]
+        run: pip install .[all]
 
       - name: Run
         run: pytest -m "unit" test/${{ matrix.topic }}
@@ -134,11 +138,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .[docstores]
+        run: pip install .[dev,docstores]
 
       - name: Run tests
         run: |
@@ -162,11 +167,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .[sql]
+        run: pip install .[dev,sql]
 
       - name: Run tests
         run: |
@@ -198,11 +204,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .[docstores]
+        run: pip install .[dev,docstores]
 
       - name: Run tests
         run: |
@@ -226,11 +233,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .
+        run: pip install .[dev]
 
       - name: Run tests
         run: |
@@ -254,11 +262,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .
+        run: pip install .[faiss,dev]
 
       - name: Run tests
         run: |
@@ -292,11 +301,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .[docstores]
+        run: pip install .[dev,docstores]
 
       - name: Run tests
         run: |
@@ -320,11 +330,12 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Install Haystack
-      run: pip install .[pinecone]
+      run: pip install .[dev,pinecone]
 
     - name: Run tests
       env:
@@ -351,8 +362,12 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
+
+    - name: Install Haystack
+      run: pip install .[all]
 
     - name: Setup Milvus
       run: |
@@ -362,7 +377,7 @@ jobs:
         sudo docker-compose ps
 
     - name: Install Haystack
-      run: pip install .[milvus]
+      run: pip install .[dev,milvus]
 
     - name: Run tests
       run: |
@@ -386,11 +401,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .
+        run: pip install .[dev]
 
       - name: Run tests
         run: |
@@ -414,11 +430,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .
+        run: pip install .[dev]
 
       - name: Run tests
         run: |
@@ -442,11 +459,12 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install Haystack
-        run: pip install -U .
+        run: pip install .[dev]
 
       - name: Run tests
         run: |
@@ -480,8 +498,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ env.PYTHON_VERSION }}
 
     # TODO evaluate if tests that need these libraries are really unit tests
     - name: Install audio libraries
@@ -490,7 +509,7 @@ jobs:
         sudo apt-get install ffmpeg
 
     - name: Install Haystack
-      run: pip install .[audio]
+      run: pip install .[all]
 
     - name: Run tests
       env:
@@ -523,15 +542,14 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
+    - uses: actions/setup-python@v4
       with:
-        prefix: windows
+        python-version: ${{ env.PYTHON_VERSION }}
 
     # - name: Install sndfile (audio support) # https://github.com/libsndfile/libsndfile/releases/download/1.1.0/libsndfile-1.1.0-win64.zip
 
     - name: Install Haystack
-      run: pip install .
+      run: pip install .[all]
 
     - name: Run tests
       env:
@@ -566,22 +584,9 @@ jobs:
     steps:
     - uses: actions/checkout@v3
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
-
-    - name: Cache HF models
-      id: cache-hf-models
-      uses: actions/cache@v3
+    - uses: actions/setup-python@v4
       with:
-        path: ~/.cache/huggingface/transformers/
-        key: hf-models
-      env:
-        SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15
-
-    - name: Download models
-      if: steps.cache-hf-models.outputs.cache-hit != 'true'
-      run: |
-         python -c "from transformers import AutoModel;[AutoModel.from_pretrained(model_name) for model_name in ['vblagoje/bart_lfqa','yjernite/bart_eli5', 'vblagoje/dpr-ctx_encoder-single-lfqa-wiki', 'vblagoje/dpr-question_encoder-single-lfqa-wiki', 'facebook/dpr-question_encoder-single-nq-base', 'facebook/dpr-ctx_encoder-single-nq-base', 'elastic/distilbert-base-cased-finetuned-conll03-english', 'deepset/bert-medium-squad2-distilled']]"
+        python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Run Elasticsearch
       run: |
@@ -618,7 +623,21 @@ jobs:
         sudo apt-get install libsndfile1 ffmpeg
 
     - name: Install Haystack
-      run: pip install .
+      run: pip install .[all]
+
+    - name: Cache HF models
+      id: cache-hf-models
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/huggingface/transformers/
+        key: hf-models
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15
+
+    - name: Download models
+      if: steps.cache-hf-models.outputs.cache-hit != 'true'
+      run: |
+         python -c "from transformers import AutoModel;[AutoModel.from_pretrained(model_name) for model_name in ['vblagoje/bart_lfqa','yjernite/bart_eli5', 'vblagoje/dpr-ctx_encoder-single-lfqa-wiki', 'vblagoje/dpr-question_encoder-single-lfqa-wiki', 'facebook/dpr-question_encoder-single-nq-base', 'facebook/dpr-ctx_encoder-single-nq-base', 'elastic/distilbert-base-cased-finetuned-conll03-english', 'deepset/bert-medium-squad2-distilled']]"
 
     - name: Run tests
       env:
@@ -671,13 +690,12 @@ jobs:
       run: |
         echo "C:\Program Files\Tesseract-OCR\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
 
-    - name: Setup Python
-      uses: ./.github/actions/python_cache/
+    - uses: actions/setup-python@v4
       with:
-        prefix: windows
+        python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Install Haystack
-      run: pip install .
+      run: pip install .[all]
 
     - name: Run tests
       env:

pyproject.toml

@@ -183,8 +183,8 @@ onnx-gpu = [
   "onnxruntime_tools",
 ]
 ray = [
-  "ray>=1.9.1,<2; platform_system != 'Windows'",
-  "ray>=1.9.1,<2,!=1.12.0; platform_system == 'Windows'",  # Avoid 1.12.0 due to https://github.com/ray-project/ray/issues/24169 (fails on windows)
+  "ray[serve]>=1.9.1,<2; platform_system != 'Windows'",
+  "ray[serve]>=1.9.1,<2,!=1.12.0; platform_system == 'Windows'",  # Avoid 1.12.0 due to https://github.com/ray-project/ray/issues/24169 (fails on windows)
   "aiorwlock>=1.3.0,<2",
 ]
 colab = [