From d322beed6c5a124f1a409eaaa9cdc4621352ec6c Mon Sep 17 00:00:00 2001 From: Massimiliano Pippi Date: Thu, 11 May 2023 19:24:47 +0200 Subject: [PATCH] build: do not install 'dev' extras with 'all' (#4888) * do not install 'dev' with 'all' * some fixes around --- .github/workflows/examples-tests.yml | 2 +- .github/workflows/linting.yml | 4 +-- .github/workflows/openapi_sync.yml | 2 +- .github/workflows/tests.yml | 10 +++---- CONTRIBUTING.md | 4 +-- README.md | 44 ++++++++++++++-------------- pyproject.toml | 4 +-- 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/examples-tests.yml b/.github/workflows/examples-tests.yml index 18ce7d59f..9cd41d14f 100644 --- a/.github/workflows/examples-tests.yml +++ b/.github/workflows/examples-tests.yml @@ -40,7 +40,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install .[all] + run: pip install .[all,dev] - name: Run run: pytest examples/ diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index cccea2c57..4671dac50 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -36,7 +36,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install ".[all]" + run: pip install ".[all,dev]" - name: Mypy if: steps.files.outputs.any_changed == 'true' @@ -70,7 +70,7 @@ jobs: - name: Install Haystack run: | - pip install ".[all]" + pip install ".[all,dev]" pip install ./haystack-linter - name: Pylint diff --git a/.github/workflows/openapi_sync.yml b/.github/workflows/openapi_sync.yml index 0d1c0072c..b45e2150b 100644 --- a/.github/workflows/openapi_sync.yml +++ b/.github/workflows/openapi_sync.yml @@ -26,7 +26,7 @@ jobs: - name: Install Haystack run: | pip install --upgrade pip - pip install -U -e .[all] + pip install -U -e .[all,dev] pip install -e ./rest_api - name: Update OpenAPI specs diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0075b74d4..2d062042d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -155,7 +155,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install .[all] + run: pip install .[all,dev] - name: Run run: pytest --cov-report xml:coverage.xml --cov="haystack" -m "unit" test/${{ matrix.topic }} @@ -1051,7 +1051,7 @@ jobs: sudo apt-get install ffmpeg - name: Install Haystack - run: pip install .[all] + run: pip install .[all,dev] - name: Run tests env: @@ -1138,7 +1138,7 @@ jobs: # - name: Install sndfile (audio support) # https://github.com/libsndfile/libsndfile/releases/download/1.1.0/libsndfile-1.1.0-win64.zip - name: Install Haystack - run: pip install .[all] + run: pip install .[all,dev] - name: Run tests env: @@ -1252,7 +1252,7 @@ jobs: sudo apt-get install libsndfile1 ffmpeg - name: Install Haystack - run: pip install .[all] + run: pip install .[all,dev] - name: Cache HF models id: cache-hf-models @@ -1371,7 +1371,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Install Haystack - run: pip install .[all] + run: pip install .[all,dev] - name: Run tests env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea92675f0..2837a9328 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,7 +54,7 @@ pip install --upgrade pip # Install Haystack in editable mode pip install -e '.[dev]' ``` -Note that the `.[dev]` part is enough in many development scenarios when adding minor code fixes. However, if your changes require a schema change, then you'll need to install all dependencies with `pip install -e '.[all]' ` command. Introducing new components or changing their interface requires a schema change. +Note that the `.[dev]` part is enough in many development scenarios when adding minor code fixes. However, if your changes require a schema change, then you'll need to install all dependencies with `pip install -e '.[all,dev]' ` command. Introducing new components or changing their interface requires a schema change. This will install all the dependencies you need to work on the codebase, plus testing and formatting dependencies. Last, install the pre-commit hooks with: @@ -278,7 +278,7 @@ We formally define three scopes for tests in Haystack with different requirement - Might not be possible to run locally due to system and hardware requirements - **Goal: being confident in releasing Haystack** -> **Note**: migrating the existing tests into these new categories is still in progress. Please ask the maintainers if you are in doubt about how to +> **Note**: migrating the existing tests into these new categories is still in progress. Please ask the maintainers if you are in doubt about how to classify your tests or where to place them. If you are writing a test that depend on a document store, there are a few conventions to define on which document store diff --git a/README.md b/README.md index c165484b2..9df6d4bb4 100644 --- a/README.md +++ b/README.md @@ -71,19 +71,19 @@ An example pipeline would consist of one `Retriever` Node and one `Reader` Node. - **Continuous Learning**: Collect new training data from user feedback in production & improve your models continuously. ## Resources -| | | -| --------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 📒 [Docs](https://docs.haystack.deepset.ai) | Components, Pipeline Nodes, Guides, API Reference | +| | | +| ---------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 📒 [Docs](https://docs.haystack.deepset.ai) | Components, Pipeline Nodes, Guides, API Reference | | 💾 [Installation](https://github.com/deepset-ai/haystack#-installation) | How to install Haystack | -| 🎓 [Tutorials](https://haystack.deepset.ai/tutorials) | See what Haystack can do with our Notebooks & Scripts | -| 🎉 [Haystack Extras](https://github.com/deepset-ai/haystack-extras) | A repository that lists extra Haystack packages and components that can be installed separately. | -| 🔰 [Demos](https://github.com/deepset-ai/haystack-demos) | A repository containing Haystack demo applications with Docker Compose and a REST API | -| 🖖 [Community](https://github.com/deepset-ai/haystack#-community) | [Discord](https://haystack.deepset.ai/community/join), [Twitter](https://twitter.com/deepset_ai), [Stack Overflow](https://stackoverflow.com/questions/tagged/haystack), [GitHub Discussions](https://github.com/deepset-ai/haystack/discussions) | -| 💙 [Contributing](https://github.com/deepset-ai/haystack#-contributing) | We welcome all contributions! | -| 📊 [Benchmarks](https://haystack.deepset.ai/benchmarks/) | Speed & Accuracy of Retriever, Readers and DocumentStores | -| 🔭 [Roadmap](https://haystack.deepset.ai/overview/roadmap) | Public roadmap of Haystack | -| 📰 [Blog](https://haystack.deepset.ai/blog) | Learn about the latest with Haystack and NLP | -| ☎️ [Jobs](https://www.deepset.ai/jobs) | We're hiring! Have a look at our open positions | +| 🎓 [Tutorials](https://haystack.deepset.ai/tutorials) | See what Haystack can do with our Notebooks & Scripts | +| 🎉 [Haystack Extras](https://github.com/deepset-ai/haystack-extras) | A repository that lists extra Haystack packages and components that can be installed separately. | +| 🔰 [Demos](https://github.com/deepset-ai/haystack-demos) | A repository containing Haystack demo applications with Docker Compose and a REST API | +| 🖖 [Community](https://github.com/deepset-ai/haystack#-community) | [Discord](https://haystack.deepset.ai/community/join), [Twitter](https://twitter.com/deepset_ai), [Stack Overflow](https://stackoverflow.com/questions/tagged/haystack), [GitHub Discussions](https://github.com/deepset-ai/haystack/discussions) | +| 💙 [Contributing](https://github.com/deepset-ai/haystack#-contributing) | We welcome all contributions! | +| 📊 [Benchmarks](https://haystack.deepset.ai/benchmarks/) | Speed & Accuracy of Retriever, Readers and DocumentStores | +| 🔭 [Roadmap](https://haystack.deepset.ai/overview/roadmap) | Public roadmap of Haystack | +| 📰 [Blog](https://haystack.deepset.ai/blog) | Learn about the latest with Haystack and NLP | +| ☎️ [Jobs](https://www.deepset.ai/jobs) | We're hiring! Have a look at our open positions | ## 💾 Installation @@ -94,25 +94,25 @@ For a detailed installation guide see [the official documentation](https://docs. Use [pip](https://github.com/pypa/pip) to install a basic version of Haystack's latest release: -``` - pip install farm-haystack +```sh +pip install farm-haystack ``` -This command installs everything needed for basic Pipelines that use an Elasticsearch DocumentStore. +This command installs everything needed for basic Pipelines that use an in-memory DocumentStore. **Full Installation** -To use more advanced features, like certain DocumentStores, FileConverters, OCR, or Ray, install further dependencies. The following command installs the [latest release](https://github.com/deepset-ai/haystack/releases) of Haystack and all its dependencies: +To use more advanced features, like certain DocumentStores, FileConverters, OCR, or Ray, +you need to install further dependencies. The following command installs the [latest release](https://github.com/deepset-ai/haystack/releases) of Haystack and all its dependencies: ```bash -pip install --upgrade pip pip install 'farm-haystack[all]' ## or 'all-gpu' for the GPU-enabled dependencies ``` -If you want to try out the newest features that are not in an official release yet, you can install Haystack from the main branch. The following command installs from `main` branch with `dev` dependencies: +If you want to try out the newest features that are not in an official release yet, you can install the unstable version from the main branch with the following command: -``` -pip install git+https://github.com/deepset-ai/haystack.git@main#egg=farm-haystack[dev] +```sh +pip install git+https://github.com/deepset-ai/haystack.git@main#egg=farm-haystack ``` To be able to make changes to Haystack code, install with the following commands: @@ -127,8 +127,8 @@ cd haystack # Upgrade pip pip install --upgrade pip -# Install Haystack in editable mode -pip install -e '.[all]' +# Install Haystack in editable mode (-e) with the development tools +pip install -e '.[dev]' ``` If you want to contribute to the Haystack repo, check our [Contributor Guidelines](#💙-contributing) first. diff --git a/pyproject.toml b/pyproject.toml index f4c31de69..a956ffb58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,11 +227,11 @@ formatting = [ ] all = [ - "farm-haystack[docstores,audio,crawler,preprocessing,file-conversion,pdf,ocr,ray,dev,onnx,beir,metrics]", + "farm-haystack[docstores,audio,crawler,preprocessing,file-conversion,pdf,ocr,ray,onnx,beir,metrics]", ] all-gpu = [ # beir is incompatible with faiss-gpu: https://github.com/beir-cellar/beir/issues/71 - "farm-haystack[docstores-gpu,audio,crawler,preprocessing,file-conversion,pdf,ocr,ray,dev,onnx-gpu,metrics]", + "farm-haystack[docstores-gpu,audio,crawler,preprocessing,file-conversion,pdf,ocr,ray,onnx-gpu,metrics]", ] [project.scripts]