2023-02-23 13:03:45 +01:00

694 lines
21 KiB
YAML

name: Tests
on:
workflow_dispatch: # Activate this workflow manually
push:
branches:
- main
# release branches have the form v1.9.x
- 'v[0-9].*[0-9].x'
pull_request:
types:
- opened
- reopened
- synchronize
- ready_for_review
paths:
- "**.py"
- "pyproject.toml"
- "!.github/**/*.py"
- "!rest_api/**/*.py"
env:
HAYSTACK_TELEMETRY_VERSION: "0" # Disables telemetry
PYTEST_PARAMS: --maxfail=5 --durations=10 --suppress-no-test-exit-code
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SUITES_EXCLUDED_FROM_WINDOWS:
--ignore=test/pipelines/test_ray.py
--ignore=test/document_stores/test_knowledge_graph.py
--ignore=test/nodes/test_audio.py
--ignore=test/nodes/test_summarizer_translation.py
--ignore=test/nodes/test_summarizer.py
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
jobs:
black:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
- name: Install Black
run: |
pip install --upgrade pip
pip install .[formatting]
- name: Check status
run: |
if ! black . --check; then
git status
echo "###################################################################################################"
echo "# "
echo "# CHECK FAILED! Black found issues with your code formatting."
echo "# "
echo "# Either:"
echo "# 1. Run Black locally before committing:"
echo "# "
echo "# pip install .[formatting]"
echo "# black ."
echo "# "
echo "# 2. Install the pre-commit hook:"
echo "# "
echo "# pre-commit install --hook-type pre-push"
echo "# "
echo "# 3. See https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md for help."
echo "# "
echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
echo "# "
echo "##################################################################################################"
exit 1
fi
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.ref == 'refs/heads/main'
unit-tests:
name: Unit / ${{ matrix.topic }} / ${{ matrix.os }}
needs: black
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
- macos-latest
topic:
- document_stores
- nodes
- agents
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install .[all]
- name: Run
run: pytest -m "unit" test/${{ matrix.topic }}
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-elasticsearch:
name: Integration / Elasticsearch / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
services:
elasticsearch:
image: elasticsearch:7.17.6
env:
discovery.type: "single-node"
ES_JAVA_OPTS: "-Xms128m -Xmx256m"
ports:
- 9200:9200
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .[docstores]
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_elasticsearch.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-sql:
name: Integration / SQL / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest,macos-latest,windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .[sql]
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_sql.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-opensearch:
name: Integration / Opensearch / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
services:
opensearch:
image: opensearchproject/opensearch:1.3.5
env:
discovery.type: "single-node"
ES_JAVA_OPTS: "-Xms128m -Xmx256m"
ports:
- 9200:9200
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .[docstores]
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_opensearch.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-dc:
name: Integration / dC / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest,macos-latest,windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_deepsetcloud.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-faiss:
name: Integration / faiss / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest,macos-latest,windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_faiss.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-weaviate:
name: Integration / Weaviate / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
services:
weaviate:
image: semitechnologies/weaviate:1.17.2
env:
AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: "true"
PERSISTENCE_DATA_PATH: "/var/lib/weaviate"
ENABLE_EXPERIMENTAL_BM25: "true"
DISK_USE_READONLY_PERCENTAGE: 95
ports:
- 8080:8080
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .[docstores]
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_weaviate.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-pinecone:
name: Integration / pinecone / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest,macos-latest,windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install .[pinecone]
- name: Run tests
env:
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_pinecone.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-milvus:
name: Integration / Milvus / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Setup Milvus
run: |
cd ../../ # Avoid causing permission issues on hashFiles later by creating unreadable folders like "volumes"
wget https://github.com/milvus-io/milvus/releases/download/v2.0.0/milvus-standalone-docker-compose.yml -O docker-compose.yml
sudo docker-compose up -d
sudo docker-compose ps
- name: Install Haystack
run: pip install .[milvus]
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_milvus.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-memory:
name: Integration / memory / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest,macos-latest,windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .
- name: Run tests
run: |
pytest --maxfail=5 -m "document_store and integration" test/document_stores/test_memory.py
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-agents:
name: Integration / Agents / ${{ matrix.os }}
needs:
- unit-tests
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest,windows-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Install Haystack
run: pip install -U .
- name: Run tests
run: |
pytest --maxfail=5 -m "integration" test/agents
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
#
# IMPORTANT: the following steps need to be revisited PLEASE DO NOT ADD ANYTHING TO THE JOBS BELOW!
#
unit-tests-linux:
needs: black
strategy:
fail-fast: false # Avoid cancelling the others if one of these fails
matrix:
folder:
- "nodes"
- "pipelines"
- "modeling"
- "others"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
# TODO Let's try to remove this one from the unit tests
- name: Install pdftotext
run: wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.04.tar.gz && tar -xvf xpdf-tools-linux-4.04.tar.gz && sudo cp xpdf-tools-linux-4.04/bin64/pdftotext /usr/local/bin
# TODO evaluate if tests that need these libraries are really unit tests
- name: Install audio libraries
run: |
sudo apt-get update
sudo apt-get install libsndfile1 ffmpeg
- name: Install Haystack
run: pip install .[audio]
- name: Run tests
env:
TOKENIZERS_PARALLELISM: 'false'
# as confusing as it seems, we skip tests marked as unit here as it means they have been migrated to the right job already
run: |
pytest ${{ env.PYTEST_PARAMS }} -m "not elasticsearch and not faiss and not milvus and not weaviate and not pinecone and not integration and not unit" test/${{ matrix.folder }} --document_store_type=memory
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
unit-tests-windows:
needs: black
strategy:
fail-fast: false # Avoid cancelling the others if one of these fails
matrix:
folder:
- "nodes"
- "pipelines"
- "modeling"
runs-on: windows-latest
if: contains(github.event.pull_request.labels.*.name, 'topic:windows') || !github.event.pull_request.draft
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
with:
prefix: windows
- name: Install pdftotext
run: |
choco install xpdf-utils
choco install openjdk11
refreshenv
# - name: Install sndfile (audio support) # https://github.com/libsndfile/libsndfile/releases/download/1.1.0/libsndfile-1.1.0-win64.zip
- name: Install Haystack
run: pip install .
- name: Run tests
env:
TOKENIZERS_PARALLELISM: 'false'
# as confusing as it seems, we skip tests marked as unit here as it means they have been migrated to the right job already
run: |
pytest ${{ env.PYTEST_PARAMS }} -m "not elasticsearch and not faiss and not milvus and not weaviate and not pinecone and not integration and not unit" ${{ env.SUITES_EXCLUDED_FROM_WINDOWS }} test/${{ matrix.folder }} --document_store_type=memory
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-linux:
needs:
- unit-tests-linux
timeout-minutes: 60
strategy:
fail-fast: false # Avoid cancelling the others if one of these fails
matrix:
folder:
- "nodes"
- "pipelines"
- "modeling"
- "others"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Python
uses: ./.github/actions/python_cache/
- name: Cache HF models
id: cache-hf-models
uses: actions/cache@v3
with:
path: ~/.cache/huggingface/transformers/
key: hf-models
env:
SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15
- name: Download models
if: steps.cache-hf-models.outputs.cache-hit != 'true'
run: |
python -c "from transformers import AutoModel;[AutoModel.from_pretrained(model_name) for model_name in ['vblagoje/bart_lfqa','yjernite/bart_eli5', 'vblagoje/dpr-ctx_encoder-single-lfqa-wiki', 'vblagoje/dpr-question_encoder-single-lfqa-wiki', 'facebook/dpr-question_encoder-single-nq-base', 'facebook/dpr-ctx_encoder-single-nq-base', 'elastic/distilbert-base-cased-finetuned-conll03-english', 'deepset/bert-medium-squad2-distilled']]"
- name: Run Elasticsearch
run: |
docker run -d -p 9200:9200 -e "discovery.type=single-node" -e "ES_JAVA_OPTS=-Xms128m -Xmx256m" elasticsearch:7.9.2
- name: Run Opensearch
run: |
docker run -d -p 9201:9200 -p 9600:9600 -e "discovery.type=single-node" opensearchproject/opensearch:1.3.5
- name: Run Milvus
run: |
cd ../../ # Avoid causing permission issues on hashFiles later by creating unreadable folders like "volumes"
wget https://github.com/milvus-io/milvus/releases/download/v2.0.0/milvus-standalone-docker-compose.yml -O docker-compose.yml
sudo docker-compose up -d
sudo docker-compose ps
- name: Run Weaviate
run: docker run -d -p 8080:8080 --name haystack_test_weaviate --env AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED='true' --env PERSISTENCE_DATA_PATH='/var/lib/weaviate' --env ENABLE_EXPERIMENTAL_BM25='true' --env DISK_USE_READONLY_PERCENTAGE='95' semitechnologies/weaviate:1.17.2
- name: Run Apache Tika
run: docker run -d -p 9998:9998 -e "TIKA_CHILD_JAVA_OPTS=-JXms128m" -e "TIKA_CHILD_JAVA_OPTS=-JXmx128m" apache/tika:1.28.4
- name: Run Parsr
run: docker run -d -p 3001:3001 axarev/parsr:v1.2.2
- name: Install pdftotext
run: wget --no-check-certificate https://dl.xpdfreader.com/xpdf-tools-linux-4.04.tar.gz && tar -xvf xpdf-tools-linux-4.04.tar.gz && sudo cp xpdf-tools-linux-4.04/bin64/pdftotext /usr/local/bin
- name: Install tesseract
run: |
sudo apt update
sudo apt-get install tesseract-ocr libtesseract-dev poppler-utils
- name: Install audio libraries
run: |
sudo apt-get update
sudo apt-get install libsndfile1 ffmpeg
- name: Install Haystack
run: pip install .
- name: Run tests
env:
TOKENIZERS_PARALLELISM: 'false' # Avoid logspam by tokenizers
# we add "and not document_store" to exclude the tests that were ported to the new strategy
run: |
pytest ${{ env.PYTEST_PARAMS }} -m "integration and not document_store" test/${{ matrix.folder }}
- name: Dump docker logs on failure
if: failure()
uses: jwalton/gh-docker-logs@v1
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
integration-tests-windows:
needs:
- unit-tests-windows
runs-on: windows-latest
if: contains(github.event.pull_request.labels.*.name, 'topic:windows') || !github.event.pull_request.draft
timeout-minutes: 30
strategy:
fail-fast: false # Avoid cancelling the others if one of these fails
matrix:
folder:
- "nodes"
- "pipelines"
- "modeling"
- "others"
steps:
- uses: actions/checkout@v3
- name: Install dependencies
run: |
choco install --no-progress xpdf-utils
choco install --no-progress tesseract
choco install --no-progress openjdk --version=11.0.2.01
refreshenv
choco install --no-progress elasticsearch --version=7.9.2
refreshenv
Get-Service elasticsearch-service-x64 | Start-Service
- name: Setup Python
uses: ./.github/actions/python_cache/
with:
prefix: windows
- name: Install Haystack
run: pip install .
- name: Run tests
env:
TOKENIZERS_PARALLELISM: 'false' # Avoid logspam by tokenizers
# FIXME many tests are disabled here!
run: |
pytest ${{ env.PYTEST_PARAMS }} -m "integration and not tika and not graphdb" ${{ env.SUITES_EXCLUDED_FROM_WINDOWS }} test/${{ matrix.folder }} --document_store_type=memory,faiss,elasticsearch
- uses: act10ns/slack@v2
with:
status: ${{ job.status }}
channel: '#haystack'
if: failure() && github.repository_owner == 'deepset-ai' && github.ref == 'refs/heads/main'
catch-all:
name: Catch-all check
runs-on: ubuntu-latest
# This job will be executed only after all the other tests
# are successful.
# This way we'll be able to mark only this test as required
# and skip it accordingly.
needs:
- integration-tests-elasticsearch
- integration-tests-sql
- integration-tests-opensearch
- integration-tests-dc
- integration-tests-faiss
- integration-tests-weaviate
- integration-tests-pinecone
- integration-tests-milvus
- integration-tests-memory
- integration-tests-linux
- integration-tests-windows
steps:
- name: Finisher
run: echo "Finish him!"