From 0f605118d99bc2843000d16f9c5aa2969b526ff9 Mon Sep 17 00:00:00 2001 From: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> Date: Fri, 17 Mar 2023 13:55:07 +0100 Subject: [PATCH] ci: remove python_cache internal action (#4429) --- .github/actions/python_cache/action.yml | 51 ------- .github/workflows/e2e.yml | 7 +- .github/workflows/examples-tests.yml | 6 +- .github/workflows/linting.yml | 19 ++- .github/workflows/minor_version_release.yml | 13 +- .github/workflows/openapi_sync.yml | 8 +- .github/workflows/rest_api_tests.yml | 11 +- .github/workflows/tests.yml | 144 +++++++++++--------- pyproject.toml | 4 +- 9 files changed, 132 insertions(+), 131 deletions(-) delete mode 100644 .github/actions/python_cache/action.yml diff --git a/.github/actions/python_cache/action.yml b/.github/actions/python_cache/action.yml deleted file mode 100644 index 207f1a6fa..000000000 --- a/.github/actions/python_cache/action.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Python Cache -description: Caches a full installation of Haystack dependencies - -inputs: - prefix: - description: 'prefix for the cache (to distinguish OS)' - required: true - default: 'linux' - pythonVersion: - description: 'Python version to use' - required: true - default: "3.8" - -runs: - using: "composite" - steps: - - - name: Get date - shell: bash - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v3 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.head_ref }} - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.pythonVersion }} - - - name: Cache Python path - id: cache-python-env - uses: actions/cache@v3 - with: - path: ${{ env.pythonLocation }} - # The cache will be rebuild every day and at every change of the dependency files - key: ${{ inputs.prefix }}-py${{ inputs.pythonVersion }}-${{ env.date }}-${{ hashFiles('**/pyproject.toml') }} - - - name: Install dependencies - # NOTE: this action runs only on cache miss to refresh the dependencies and make sure the next Haystack install will be faster. - # Do not rely on this step to install Haystack itself! - # The cache can last way longer than a specific action's run, so older Haystack version could be carried over and the tests will run - # against the cached code instead of the new changes. - if: steps.cache-python-env.outputs.cache-hit != 'true' - shell: bash - run: | - python -m pip install --upgrade pip - pip install .[all,alfalfa] - pip install rest_api/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 96df8ffda..17e1310a4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -4,6 +4,8 @@ name: end-to-end on: workflow_dispatch: +env: + PYTHON_VERSION: "3.8" jobs: e2e: @@ -19,8 +21,9 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Cache HF models id: cache-hf-models diff --git a/.github/workflows/examples-tests.yml b/.github/workflows/examples-tests.yml index 8da8c73ba..977c5d5b2 100644 --- a/.github/workflows/examples-tests.yml +++ b/.github/workflows/examples-tests.yml @@ -16,6 +16,7 @@ on: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + PYTHON_VERSION: "3.8" jobs: tests: @@ -33,8 +34,9 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack run: pip install .[all] diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index a3fcd16c9..9080084a2 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -7,6 +7,9 @@ on: - "**.py" - "**/pyproject.toml" +env: + PYTHON_VERSION: "3.8" + jobs: mypy: runs-on: ubuntu-latest @@ -28,8 +31,12 @@ jobs: test/** rest_api/test/** - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Haystack + run: pip install ".[all]" - name: Mypy if: steps.files.outputs.any_changed == 'true' @@ -57,8 +64,12 @@ jobs: test/** rest_api/test/** - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Haystack + run: pip install ".[all]" - name: Pylint if: steps.files.outputs.any_changed == 'true' diff --git a/.github/workflows/minor_version_release.yml b/.github/workflows/minor_version_release.yml index bb3372eb2..cad80ba1e 100644 --- a/.github/workflows/minor_version_release.yml +++ b/.github/workflows/minor_version_release.yml @@ -3,6 +3,9 @@ name: Minor Version Release on: workflow_dispatch: +env: + PYTHON_VERSION: "3.8" + jobs: sync: runs-on: ubuntu-latest @@ -10,9 +13,6 @@ jobs: - name: Checkout this repo uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ - - name: Define all versions id: versions shell: bash @@ -42,6 +42,13 @@ jobs: git push -u origin bump-version gh pr create -B main -H bump-version --title 'Bump unstable version' --body 'Part of the release process' --label 'ignore-for-release-notes' + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install release_docs.py dependencies + run: pip install requests + # Note that patch versions all sync to the one readme minor version # e.g. Haystack 1.9.1 and 1.9.2 both map to Readme 1.9 - name: Release Readme version diff --git a/.github/workflows/openapi_sync.yml b/.github/workflows/openapi_sync.yml index 418af0962..0d1c0072c 100644 --- a/.github/workflows/openapi_sync.yml +++ b/.github/workflows/openapi_sync.yml @@ -7,14 +7,18 @@ on: # release branches have the form v1.9.x - "v[0-9].*[0-9].x" +env: + PYTHON_VERSION: "3.8" + jobs: run: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install dependencies run: sudo apt update && sudo apt-get install libsndfile1 ffmpeg diff --git a/.github/workflows/rest_api_tests.yml b/.github/workflows/rest_api_tests.yml index 65be320da..c1f627913 100644 --- a/.github/workflows/rest_api_tests.yml +++ b/.github/workflows/rest_api_tests.yml @@ -17,6 +17,9 @@ on: - "rest_api/**.py" - "rest_api/pyproject.toml" +env: + PYTHON_VERSION: "3.8" + jobs: black: runs-on: ubuntu-latest @@ -24,6 +27,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Black run: | @@ -73,12 +78,14 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install REST API run: | pip install -U "./rest_api[dev]" + pip install ".[dev]" pip install . - name: Run tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c7d4ffb3c..088460980 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,6 +32,7 @@ env: --ignore=test/nodes/test_summarizer.py OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} + PYTHON_VERSION: "3.8" jobs: @@ -42,6 +43,8 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Black run: | @@ -99,11 +102,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install .[all,alfalfa] + run: pip install .[all] - name: Run run: pytest -m "unit" test/${{ matrix.topic }} @@ -134,11 +138,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U .[docstores] + run: pip install .[dev,docstores] - name: Run tests run: | @@ -162,11 +167,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U .[sql] + run: pip install .[dev,sql] - name: Run tests run: | @@ -198,11 +204,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U .[docstores] + run: pip install .[dev,docstores] - name: Run tests run: | @@ -226,11 +233,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U . + run: pip install .[dev] - name: Run tests run: | @@ -254,11 +262,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U . + run: pip install .[faiss,dev] - name: Run tests run: | @@ -292,11 +301,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U .[docstores] + run: pip install .[dev,docstores] - name: Run tests run: | @@ -320,11 +330,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install .[pinecone] + run: pip install .[dev,pinecone] - name: Run tests env: @@ -351,8 +362,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Haystack + run: pip install .[all] - name: Setup Milvus run: | @@ -362,7 +377,7 @@ jobs: sudo docker-compose ps - name: Install Haystack - run: pip install .[milvus] + run: pip install .[dev,milvus] - name: Run tests run: | @@ -386,11 +401,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U . + run: pip install .[dev] - name: Run tests run: | @@ -414,11 +430,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U . + run: pip install .[dev] - name: Run tests run: | @@ -442,11 +459,12 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install -U . + run: pip install .[dev] - name: Run tests run: | @@ -480,8 +498,9 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_VERSION }} # TODO evaluate if tests that need these libraries are really unit tests - name: Install audio libraries @@ -490,7 +509,7 @@ jobs: sudo apt-get install ffmpeg - name: Install Haystack - run: pip install .[audio] + run: pip install .[all] - name: Run tests env: @@ -523,15 +542,14 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 with: - prefix: windows + python-version: ${{ env.PYTHON_VERSION }} # - name: Install sndfile (audio support) # https://github.com/libsndfile/libsndfile/releases/download/1.1.0/libsndfile-1.1.0-win64.zip - name: Install Haystack - run: pip install . + run: pip install .[all] - name: Run tests env: @@ -566,22 +584,9 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: ./.github/actions/python_cache/ - - - name: Cache HF models - id: cache-hf-models - uses: actions/cache@v3 + - uses: actions/setup-python@v4 with: - path: ~/.cache/huggingface/transformers/ - key: hf-models - env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 - - - name: Download models - if: steps.cache-hf-models.outputs.cache-hit != 'true' - run: | - python -c "from transformers import AutoModel;[AutoModel.from_pretrained(model_name) for model_name in ['vblagoje/bart_lfqa','yjernite/bart_eli5', 'vblagoje/dpr-ctx_encoder-single-lfqa-wiki', 'vblagoje/dpr-question_encoder-single-lfqa-wiki', 'facebook/dpr-question_encoder-single-nq-base', 'facebook/dpr-ctx_encoder-single-nq-base', 'elastic/distilbert-base-cased-finetuned-conll03-english', 'deepset/bert-medium-squad2-distilled']]" + python-version: ${{ env.PYTHON_VERSION }} - name: Run Elasticsearch run: | @@ -618,7 +623,21 @@ jobs: sudo apt-get install libsndfile1 ffmpeg - name: Install Haystack - run: pip install . + run: pip install .[all] + + - name: Cache HF models + id: cache-hf-models + uses: actions/cache@v3 + with: + path: ~/.cache/huggingface/transformers/ + key: hf-models + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 15 + + - name: Download models + if: steps.cache-hf-models.outputs.cache-hit != 'true' + run: | + python -c "from transformers import AutoModel;[AutoModel.from_pretrained(model_name) for model_name in ['vblagoje/bart_lfqa','yjernite/bart_eli5', 'vblagoje/dpr-ctx_encoder-single-lfqa-wiki', 'vblagoje/dpr-question_encoder-single-lfqa-wiki', 'facebook/dpr-question_encoder-single-nq-base', 'facebook/dpr-ctx_encoder-single-nq-base', 'elastic/distilbert-base-cased-finetuned-conll03-english', 'deepset/bert-medium-squad2-distilled']]" - name: Run tests env: @@ -671,13 +690,12 @@ jobs: run: | echo "C:\Program Files\Tesseract-OCR\" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - name: Setup Python - uses: ./.github/actions/python_cache/ + - uses: actions/setup-python@v4 with: - prefix: windows + python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install . + run: pip install .[all] - name: Run tests env: diff --git a/pyproject.toml b/pyproject.toml index eae75d507..79f56f38e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -183,8 +183,8 @@ onnx-gpu = [ "onnxruntime_tools", ] ray = [ - "ray>=1.9.1,<2; platform_system != 'Windows'", - "ray>=1.9.1,<2,!=1.12.0; platform_system == 'Windows'", # Avoid 1.12.0 due to https://github.com/ray-project/ray/issues/24169 (fails on windows) + "ray[serve]>=1.9.1,<2; platform_system != 'Windows'", + "ray[serve]>=1.9.1,<2,!=1.12.0; platform_system == 'Windows'", # Avoid 1.12.0 due to https://github.com/ray-project/ray/issues/24169 (fails on windows) "aiorwlock>=1.3.0,<2", ] colab = [