# If you change this name also do it in ci_metrics.yml name: Slow Integration Tests # The workflow will always run, but the actual tests will only execute when: # - The workflow is triggered manually # - The workflow is scheduled # - The PR has the "run-slow-tests" label # - The push is to a release branch # - There are changes to relevant files. # Note: If no conditions are met, the workflow will complete successfully without running tests # to satisfy Branch Protection rules. env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} HF_API_TOKEN: ${{ secrets.HUGGINGFACE_API_KEY }} PYTHON_VERSION: "3.9" HATCH_VERSION: "1.14.1" HAYSTACK_MPS_ENABLED: false on: workflow_dispatch: # Activate this workflow manually schedule: - cron: "0 0 * * *" push: branches: # release branches have the form v1.9.x - "v[0-9].*[0-9].x" pull_request: types: - opened - reopened - synchronize - labeled - unlabeled jobs: check-if-changed: # This job checks if the relevant files have been changed. # We check for changes in the check-if-changed job instead of using paths/paths-ignore at workflow level. # This ensures the "Slow Integration Tests completed" job always runs, which is required by Branch Protection rules. name: Check if changed runs-on: ubuntu-latest permissions: pull-requests: read # Specifying outputs is not needed to make the job work, but only to comply with actionlint outputs: changes: ${{ steps.changes.outputs.changes }} steps: - uses: actions/checkout@v4 - name: Check for changed code id: changes uses: dorny/paths-filter@v3 with: # List of Python files that trigger slow integration tests when modified filters: | changes: - "haystack/components/audio/whisper_local.py" - "haystack/components/classifiers/zero_shot_document_classifier.py" - "haystack/components/converters/tika.py" - "haystack/components/embedders/hugging_face_api_document_embedder.py" - "haystack/components/embedders/hugging_face_api_text_embedder.py" - "haystack/components/embedders/sentence_transformers_text_embedder.py" - "haystack/components/evaluators/sas_evaluator.py" - "haystack/components/generators/chat/hugging_face_api.py" - "haystack/components/generators/chat/hugging_face_local.py" - "haystack/components/generators/hugging_face_api.py" - "haystack/components/generators/hugging_face_local_generator.py" - "haystack/components/rankers/sentence_transformers_diversity.py" - "haystack/components/rankers/sentence_transformers_similarity.py" - "haystack/components/rankers/transformers_similarity.py" - "haystack/components/readers/extractive.py" - "haystack/components/routers/transformers_text_router.py" - "haystack/components/routers/zero_shot_text_router.py" - "test/components/audio/test_whisper_local.py" - "test/components/classifiers/test_zero_shot_document_classifier.py" - "test/components/converters/test_tika_doc_converter.py" - "test/components/embedders/test_hugging_face_api_document_embedder.py" - "test/components/embedders/test_hugging_face_api_text_embedder.py" - "test/components/embedders/test_sentence_transformers_text_embedder.py" - "test/components/evaluators/test_sas_evaluator.py" - "test/components/generators/chat/test_hugging_face_api.py" - "test/components/generators/chat/test_hugging_face_local.py" - "test/components/generators/test_hugging_face_api.py" - "test/components/generators/test_hugging_face_local_generator.py" - "test/components/rankers/test_sentence_transformers_diversity.py" - "test/components/rankers/test_sentence_transformers_similarity.py" - "test/components/rankers/test_transformers_similarity.py" - "test/components/readers/test_extractive.py" - "test/components/routers/test_transformers_text_router.py" - "test/components/routers/test_zero_shot_text_router.py" slow-integration-tests: name: Slow Tests / ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: check-if-changed timeout-minutes: 30 # Run tests if: manual trigger, scheduled, PR has label, release branch, or relevant files changed if: | github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-slow-tests')) || (github.event_name == 'push' && github.ref == 'refs/heads/v[0-9].*[0-9].x') || (needs.check-if-changed.outputs.changes == 'true') strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] include: - os: ubuntu-latest install_cmd: "sudo apt update && sudo apt install ffmpeg" - os: macos-latest install_cmd: "brew install ffmpeg" - os: windows-latest install_cmd: "echo 'No additional dependencies needed'" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "${{ env.PYTHON_VERSION }}" - name: Install Hatch id: hatch shell: bash run: | pip install hatch==${{ env.HATCH_VERSION }} - name: Run Tika if: matrix.os == 'ubuntu-latest' run: | docker run -d -p 9998:9998 apache/tika:2.9.0.0 - name: Install Whisper dependencies shell: bash run: ${{ matrix.install_cmd }} - name: Run tests run: hatch run test:integration-only-slow - name: Send event to Datadog for nightly failures if: failure() && github.event_name == 'schedule' uses: masci/datadog@v1 with: api-key: ${{ secrets.CORE_DATADOG_API_KEY }} api-url: https://api.datadoghq.eu events: | - title: "${{ github.workflow }} workflow" text: "Job ${{ github.job }} in branch ${{ github.ref_name }}" alert_type: "error" source_type_name: "Github" host: ${{ github.repository_owner }} tags: - "project:${{ github.repository }}" - "job:${{ github.job }}" - "run_id:${{ github.run_id }}" - "workflow:${{ github.workflow }}" - "branch:${{ github.ref_name }}" - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" slow-integration-tests-completed: # This job always runs and succeeds if all tests succeed or are skipped. It is required by Branch Protection rules. name: Slow Integration Tests completed runs-on: ubuntu-latest if: ${{ always() }} needs: slow-integration-tests steps: - name: Mark tests as completed run: | if [ "${{ needs.slow-integration-tests.result }}" = "failure" ]; then echo "Slow Integration Tests failed!" exit 1 else echo "Slow Integration Tests completed!" fi