ci: unify dependency management + hatch scripts (#7079)

* ci: unify dependency management + hatch scripts

* ci: migrate readme sync

* build: migrate snippets

* ci: pin hatch

* ci: make Python version more explicit + quote

* ci: add scripts with parameters to hatch

---------

Co-authored-by: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com>

.github/workflows/docstring_labeler.yml

@@ -5,6 +5,9 @@ on:
     paths:
       - "haystack/**/*.py"
 
+env:
+  PYTHON_VERSION: "3.11"
+
 jobs:
   label:
     runs-on: ubuntu-latest
@@ -24,7 +27,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Get docstrings
         id: base-docstrings

.github/workflows/e2e.yml

@@ -18,6 +18,7 @@ on:
 env:
   PYTHON_VERSION: "3.8"
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   run:
@@ -28,15 +29,10 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: ${{ env.PYTHON_VERSION }}
+        python-version: "${{ env.PYTHON_VERSION }}"
 
-    - name: Install dependencies
-      run: |
-        sudo apt-get update
-        sudo apt install ffmpeg  # for local Whisper tests
-
-    - name: Install Haystack and the dependencies needed for tests
-      run: pip install -r test/test_requirements.txt
+    - name: Install Hatch
+      run: pip install hatch==${{ env.HATCH_VERSION }}
 
     - name: Run tests
-      run: pytest e2e
+      run: hatch run test:e2e

.github/workflows/license_compliance.yml

@@ -11,6 +11,7 @@ on:
 
 env:
   CORE_DATADOG_API_KEY: ${{ secrets.CORE_DATADOG_API_KEY }}
+  PYTHON_VERSION: "3.10"
 
 jobs:
   license_check_direct:
@@ -25,7 +26,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Get direct dependencies
         run: |

.github/workflows/linting.yml

@@ -12,6 +12,7 @@ on:
 
 env:
   PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   mypy:
@@ -35,19 +36,16 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Haystack
-        run: pip install .[dev]
-
-      - name: Install the dependencies needed for tests
-        run: pip install -r test/test_requirements.txt
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Mypy
         if: steps.files.outputs.any_changed == 'true'
         run: |
           mkdir .mypy_cache
-          mypy --install-types --non-interactive --cache-dir=.mypy_cache/ ${{ steps.files.outputs.all_changed_files }}
+          hatch run test:types ${{ steps.files.outputs.all_changed_files }}
 
   pylint:
     runs-on: ubuntu-latest
@@ -68,12 +66,12 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Haystack and the dependencies needed for tests
-        run: pip install -r test/test_requirements.txt
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Pylint
         if: steps.files.outputs.any_changed == 'true'
         run: |
-          pylint -ry -j 0 ${{ steps.files.outputs.all_changed_files }}
+          hatch run test:lint ${{ steps.files.outputs.all_changed_files }}

.github/workflows/minor_version_release.yml

@@ -49,7 +49,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install release_docs.py dependencies
         run: pip install requests

.github/workflows/pypi_release.yml

@@ -5,6 +5,9 @@ on:
     tags:
       - "v[0-9].[0-9]+.[0-9]+*"
 
+env:
+  HATCH_VERSION: "1.9.3"
+
 jobs:
   release-on-pypi:
     runs-on: ubuntu-latest
@@ -14,7 +17,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install Hatch
-        run: pip install hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Build Haystack
         run: hatch build

.github/workflows/readme_sync.yml

@@ -8,6 +8,10 @@ on:
     branches:
       - main
 
+env:
+  HATCH_VERSION: "1.9.3"
+  PYTHON_VERSION: "3.10"
+
 jobs:
   sync:
     runs-on: ubuntu-latest
@@ -18,12 +22,10 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -U haystack-pydoc-tools
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Generate API docs
         env:
@@ -31,7 +33,7 @@ jobs:
           # from Readme.io as we need them to associate the slug
           # in config files with their id.
           README_API_KEY: ${{ secrets.README_API_KEY }}
-        run: ./.github/utils/pydoc-markdown.sh
+        run: hatch run readme:sync
 
       - name: Sync docs with 2.0
         # Sync the docs to the `2.0` version on Readme

.github/workflows/snippets_tests.yml

@@ -17,6 +17,7 @@ on:
 env:
   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
   PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   black:
@@ -26,7 +27,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install Black
         run: |
@@ -100,13 +101,10 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install snippets dependencies
-        run: |
-          pip install --upgrade pip
-          pip install "." torch
-          pip install pydantic
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Get changed files
         id: files
@@ -119,7 +117,7 @@ jobs:
         run: |
           CHANGED_FILES=${{ steps.files.outputs.all_changed_files }}
           for file in $CHANGED_FILES; do
-            python "$file"
+            hatch run snippets:python "$file"
           done
 
       - name: Calculate alert data

.github/workflows/tests.yml

@@ -29,6 +29,7 @@ env:
   AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   PYTHON_VERSION: "3.8"
+  HATCH_VERSION: "1.9.3"
 
 jobs:
   black:
@@ -38,19 +39,13 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Black
-        run: |
-          pip install --upgrade pip
-          pip install .[dev]
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
 
       - name: Check status
-        run: |
-          if ! black . --check; then
-            git status
-            exit 1
-          fi
+        run: hatch run default:format-check
 
       - name: Calculate alert data
         id: calculator
@@ -96,10 +91,14 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: Install Haystack and the dependencies needed for tests
-        run: pip install -r test/test_requirements.txt
+      - name: Install Hatch
+        run: pip install hatch==${{ env.HATCH_VERSION }}
+
+      - name: Install dependencies
+        # To actually install and sync the dependencies
+        run: hatch run test:pip list
 
       - uses: actions/cache@v4
         with:
@@ -122,7 +121,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Restore Python dependencies
         uses: actions/cache/restore@v4
@@ -131,7 +130,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test
+        run: hatch run test:unit
 
       - name: Coveralls
         # We upload only coverage for ubuntu as handling both os
@@ -186,7 +185,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install dependencies
         run: |
@@ -200,7 +199,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --maxfail=5 -m "integration" test
+        run: hatch run test:integration
 
       - name: Calculate alert data
         id: calculator
@@ -245,7 +244,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Install dependencies
         run: |
@@ -258,7 +257,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --maxfail=5 -m "integration" test -k 'not tika'
+        run: hatch run test:integration-mac
 
       - name: Calculate alert data
         id: calculator
@@ -300,7 +299,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
+          python-version: "${{ env.PYTHON_VERSION }}"
 
       - name: Restore Python dependencies
         uses: actions/cache/restore@v4
@@ -309,7 +308,7 @@ jobs:
           key: pip-${{ runner.os }}-${{ github.run_id }}-${{ hashFiles('pyproject.toml', 'test/test_requirements.txt') }}
 
       - name: Run
-        run: pytest --maxfail=5 -m "integration" test -k 'not tika'
+        run: hatch run test:integration-windows
 
       - name: Calculate alert data
         id: calculator

pyproject.toml

@@ -61,8 +61,8 @@ dependencies = [
   "boilerpy3", # Fulltext extraction from HTML pages
 ]
 
-[project.optional-dependencies]
-dev = [
+[tool.hatch.envs.default]
+dependencies = [
   "pre-commit",
   # Type check
   "mypy",
@@ -91,6 +91,63 @@ dev = [
   "black[jupyter]~=23.0",
 ]
 
+[tool.hatch.envs.default.scripts]
+format = "black ."
+format-check = "black --check ."
+
+[tool.hatch.envs.test]
+extra-dependencies = [
+  "transformers[torch,sentencepiece]==4.37.2",  # ExtractiveReader, TransformersSimilarityRanker, LocalWhisperTranscriber, HFGenerators...
+  "spacy>=3.7,<3.8",  # NamedEntityExtractor
+  "spacy-curated-transformers>=0.2,<=0.3",  # NamedEntityExtractor
+  "en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.7.3/en_core_web_trf-3.7.3-py3-none-any.whl",  # NamedEntityExtractor
+
+  # Converters
+  "pypdf",  # PyPDFConverter
+  "markdown-it-py",  # MarkdownToDocument
+  "mdit_plain",  # MarkdownToDocument
+  "tika",  # TikaDocumentConverter
+  "azure-ai-formrecognizer>=3.2.0b2",  # AzureOCRDocumentConverter
+  "langdetect",  # TextLanguageRouter and DocumentLanguageClassifier
+  "sentence-transformers>=2.2.0",  # SentenceTransformersTextEmbedder and SentenceTransformersDocumentEmbedder
+  "openai-whisper>=20231106",  # LocalWhisperTranscriber
+
+  # OpenAPI
+  "jsonref",  # OpenAPIServiceConnector, OpenAPIServiceToFunctions
+  "openapi3",
+
+  # Validation
+  "jsonschema",
+
+  # Tracing
+  "opentelemetry-sdk",
+  "ddtrace",
+]
+
+[tool.hatch.envs.test.scripts]
+e2e = "pytest e2e"
+unit = 'pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" test'
+integration = 'pytest --maxfail=5 -m "integration" test'
+integration-mac = 'pytest --maxfail=5 -m "integration" test -k "not tika"'
+integration-windows = 'pytest --maxfail=5 -m "integration" test -k "not tika"'
+types = "mypy --install-types --non-interactive --cache-dir=.mypy_cache/ {args:haystack}"
+lint = "pylint -ry -j 0 {args:haystack}"
+
+[tool.hatch.envs.readme]
+detached = true  # To avoid installing the dependencies from the default environment
+dependencies = [
+  "haystack-pydoc-tools",
+]
+
+[tool.hatch.envs.readme.scripts]
+sync = "./.github/utils/pydoc-markdown.sh"
+
+[tool.hatch.envs.snippets]
+extra-dependencies = [
+  "torch",
+  "pydantic",
+]
+
 [project.urls]
 "CI: GitHub" = "https://github.com/deepset-ai/haystack/actions"
 "Docs: RTD" = "https://haystack.deepset.ai/overview/intro"

test/test_requirements.txt

@@ -1,30 +0,0 @@
-.[dev]
-
-# Package                                        Components
-
-transformers[torch,sentencepiece]==4.37.2       # ExtractiveReader, TransformersSimilarityRanker, LocalWhisperTranscriber, HFGenerators...
-spacy>=3.7,<3.8                                 # NamedEntityExtractor
-spacy-curated-transformers>=0.2,<=0.3           # NamedEntityExtractor
-https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.7.3/en_core_web_trf-3.7.3-py3-none-any.whl # NamedEntityExtractor
-
-# Converters
-pypdf                                           # PyPDFConverter
-markdown-it-py                                  # MarkdownToDocument
-mdit_plain                                      # MarkdownToDocument
-tika                                            # TikaDocumentConverter
-azure-ai-formrecognizer>=3.2.0b2                # AzureOCRDocumentConverter
-
-langdetect                                      # TextLanguageRouter and DocumentLanguageClassifier
-sentence-transformers>=2.2.0                    # SentenceTransformersTextEmbedder and SentenceTransformersDocumentEmbedder
-openai-whisper>=20231106                        # LocalWhisperTranscriber
-
-# OpenAPI
-jsonref                                        # OpenAPIServiceConnector, OpenAPIServiceToFunctions
-openapi3
-
-# Validation
-jsonschema
-
-# Tracing
-opentelemetry-sdk
-ddtrace