diff --git a/.github/utils/pydoc-markdown.sh b/.github/utils/pydoc-markdown.sh index 0fc56d72a..4e1c21f68 100755 --- a/.github/utils/pydoc-markdown.sh +++ b/.github/utils/pydoc-markdown.sh @@ -1,7 +1,10 @@ #!/bin/bash set -e # Fails on any error in the following loop +export PYTHONPATH=$PWD/docs/pydoc # Make the renderers available to pydoc cd docs/_src/api/api/ -for file in ../pydoc/* ; do +mkdir temp +cd temp +for file in ../../pydoc/* ; do pydoc-markdown "$file" done diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml deleted file mode 100644 index dfc4ff771..000000000 --- a/.github/workflows/documentation.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Documentation - -on: - workflow_dispatch: # Activate this workflow manually - pull_request: - -jobs: - - api-docs: - runs-on: ubuntu-latest - steps: - - - uses: actions/checkout@v2 - - - name: Setup Python - uses: ./.github/actions/python_cache/ - - - name: Install Haystack - run: | - pip install --upgrade pip - pip install -U .[dev] - - - name: Update API documentation - run: .github/utils/pydoc-markdown.sh - - - name: Check status - run: | - if [[ `git status --porcelain` ]]; then - git status - echo "###################################################################################################" - echo "# " - echo "# CHECK FAILED! The API docs were not updated." - echo "# " - echo "# Either:" - echo "# 1. Generate the new API docs locally before committing:" - echo "# " - echo "# .github/utils/pydoc-markdown.sh" - echo "# " - echo "# 2. Install the pre-commit hook:" - echo "# " - echo "# pre-commit install --hook-type pre-push" - echo "# " - echo "# 3. See https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md for help." - echo "# " - echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues" - echo "# " - echo "###################################################################################################" - exit 1 - fi diff --git a/.github/workflows/readme_integration.yml b/.github/workflows/readme_integration.yml new file mode 100644 index 000000000..cabbfa97d --- /dev/null +++ b/.github/workflows/readme_integration.yml @@ -0,0 +1,34 @@ +name: Testing Haystack-Readme integration + +on: + push: + branches: + - main + +jobs: + sync: + runs-on: ubuntu-latest + steps: + + - name: Checkout this repo + uses: actions/checkout@v3 + + - name: Set up Python 3.8.10 + uses: actions/setup-python@v2 + with: + python-version: 3.8.10 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pydoc-markdown==4.5.1 + + - name: Generate API docs + run: ./.github/utils/pydoc-markdown.sh + + - name: Github Action (API sync) + uses: readmeio/rdme@7.2.0 + env: + README_API_KEY: ${{ secrets.README_API_KEY }} + with: + rdme: docs ./docs/_src/api/api/temp --key="$README_API_KEY" --version=1.10-unstable diff --git a/docs/_src/api/pydoc/answer-generator.yml b/docs/_src/api/pydoc/answer-generator.yml index fe87c558b..3ac7cd95e 100644 --- a/docs/_src/api/pydoc/answer-generator.yml +++ b/docs/_src/api/pydoc/answer-generator.yml @@ -12,9 +12,17 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: generator.md + type: renderers.ReadmeRenderer + excerpt: Reads a set of documents and generates an answer to a question, word by word + category: 6310ca73c622850ddd3875a2 + title: Answer Generator API + slug: answer-generator-api + order: 0 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: answer_generator_api.md + + diff --git a/docs/_src/api/pydoc/crawler.yml b/docs/_src/api/pydoc/crawler.yml index b9489d589..9fc378c4a 100644 --- a/docs/_src/api/pydoc/crawler.yml +++ b/docs/_src/api/pydoc/crawler.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: crawler.md + type: renderers.ReadmeRenderer + excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file. + category: 6310ca73c622850ddd3875a2 + title: Crawler API + slug: crawler-api + order: 10 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: crawler_api.md diff --git a/docs/_src/api/pydoc/document-classifier.yml b/docs/_src/api/pydoc/document-classifier.yml index 4070d3677..1aa7523a8 100644 --- a/docs/_src/api/pydoc/document-classifier.yml +++ b/docs/_src/api/pydoc/document-classifier.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: document_classifier.md + type: renderers.ReadmeRenderer + excerpt: Used to create predictions that are attached to documents as metadata. + category: 6310ca73c622850ddd3875a2 + title: Document Classifier API + slug: document-classifier-api + order: 20 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: document_classifier_api.md diff --git a/docs/_src/api/pydoc/document-store.yml b/docs/_src/api/pydoc/document-store.yml index 33181e307..520633756 100644 --- a/docs/_src/api/pydoc/document-store.yml +++ b/docs/_src/api/pydoc/document-store.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: document_store.md + type: renderers.ReadmeRenderer + excerpt: Stores your texts and meta data and provides them to the Retriever at query time. + category: 6310ca73c622850ddd3875a2 + title: Document Store API + slug: document-store-api + order: 30 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: document_store_api.md diff --git a/docs/_src/api/pydoc/evaluation.yml b/docs/_src/api/pydoc/evaluation.yml index cfff806d3..3753fb979 100644 --- a/docs/_src/api/pydoc/evaluation.yml +++ b/docs/_src/api/pydoc/evaluation.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: evaluation.md + type: renderers.ReadmeRenderer + excerpt: The tools needed to evaluate whole pipelines or individual Nodes. + category: 6310ca73c622850ddd3875a2 + title: Evaluation API + slug: evaluation-api + order: 40 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: evaluation_api.md diff --git a/docs/_src/api/pydoc/extractor.yml b/docs/_src/api/pydoc/extractor.yml index 5dd3add5e..b6645fb1d 100644 --- a/docs/_src/api/pydoc/extractor.yml +++ b/docs/_src/api/pydoc/extractor.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: extractor.md + type: renderers.ReadmeRenderer + excerpt: Extracts predefined entities out of a piece of text. + category: 6310ca73c622850ddd3875a2 + title: Entity Extractor API + slug: entity-extractor-api + order: 50 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: entity_extractor_api.md diff --git a/docs/_src/api/pydoc/file-classifier.yml b/docs/_src/api/pydoc/file-classifier.yml index b6fa4c94a..3d14e4849 100644 --- a/docs/_src/api/pydoc/file-classifier.yml +++ b/docs/_src/api/pydoc/file-classifier.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: file_classifier.md + type: renderers.ReadmeRenderer + excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline. + category: 6310ca73c622850ddd3875a2 + title: File Classifier API + slug: file-classifier-api + order: 60 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: file_classifier_api.md diff --git a/docs/_src/api/pydoc/file-converters.yml b/docs/_src/api/pydoc/file-converters.yml index 14b17e26f..a590f45e4 100644 --- a/docs/_src/api/pydoc/file-converters.yml +++ b/docs/_src/api/pydoc/file-converters.yml @@ -5,16 +5,22 @@ loaders: ignore_when_discovered: ['__init__'] processors: - type: filter - expression: + expression: documented_only: true do_not_filter_modules: false skip_empty_modules: true - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: file_converter.md + type: renderers.ReadmeRenderer + excerpt: Extracts text from files in different formats and cast it into the unified Document format. + category: 6310ca73c622850ddd3875a2 + title: File Converters API + slug: file-converters-api + order: 70 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: file_converters_api.md diff --git a/docs/_src/api/pydoc/other-nodes.yml b/docs/_src/api/pydoc/other.yml similarity index 51% rename from docs/_src/api/pydoc/other-nodes.yml rename to docs/_src/api/pydoc/other.yml index 0d3c4483e..d94eb6974 100644 --- a/docs/_src/api/pydoc/other-nodes.yml +++ b/docs/_src/api/pydoc/other.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: other_nodes.md + type: renderers.ReadmeRenderer + excerpt: The utility classes of Haystack. + category: 6310ca73c622850ddd3875a2 + title: Other API + slug: other-api + order: 80 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: other_api.md diff --git a/docs/_src/api/pydoc/pipelines.yml b/docs/_src/api/pydoc/pipelines.yml index 705643cca..acc8a3186 100644 --- a/docs/_src/api/pydoc/pipelines.yml +++ b/docs/_src/api/pydoc/pipelines.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: pipelines.md + type: renderers.ReadmeRenderer + excerpt: Arranges nodes in a predefined flow. + category: 6310ca73c622850ddd3875a2 + title: Pipelines API + slug: pipelines-api + order: 90 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: pipelines_api.md diff --git a/docs/_src/api/pydoc/preprocessor.yml b/docs/_src/api/pydoc/preprocessor.yml index 5481f76c1..aa7ede192 100644 --- a/docs/_src/api/pydoc/preprocessor.yml +++ b/docs/_src/api/pydoc/preprocessor.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: preprocessor.md + type: renderers.ReadmeRenderer + excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces. + category: 6310ca73c622850ddd3875a2 + title: PreProcessor API + slug: preprocessor-api + order: 100 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: preprocessor_api.md diff --git a/docs/_src/api/pydoc/primitives.yml b/docs/_src/api/pydoc/primitives.yml index b400f9319..f5d9e971a 100644 --- a/docs/_src/api/pydoc/primitives.yml +++ b/docs/_src/api/pydoc/primitives.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: primitives.md + type: renderers.ReadmeRenderer + excerpt: These are the core classes that carry data through the system. + category: 6310ca73c622850ddd3875a2 + title: Primitives API + slug: primitives-api + order: 110 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: primitives_api.md diff --git a/docs/_src/api/pydoc/pseudo-label-generator.yml b/docs/_src/api/pydoc/pseudo-label-generator.yml index f21f5a388..8043b2cd9 100644 --- a/docs/_src/api/pydoc/pseudo-label-generator.yml +++ b/docs/_src/api/pydoc/pseudo-label-generator.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: pseudo_label_generator.md + type: renderers.ReadmeRenderer + excerpt: Creates training data for dense retrievers without human annotation. + category: 6310ca73c622850ddd3875a2 + title: Pseudo Label Generator API + slug: pseudo-label-generator-api + order: 120 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: pseudo_label_generator_api.md diff --git a/docs/_src/api/pydoc/query-classifier.yml b/docs/_src/api/pydoc/query-classifier.yml index 5be82ee87..b31c513d4 100644 --- a/docs/_src/api/pydoc/query-classifier.yml +++ b/docs/_src/api/pydoc/query-classifier.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: query_classifier.md + type: renderers.ReadmeRenderer + excerpt: Distinguishes between keyword, question and statements queries. + category: 6310ca73c622850ddd3875a2 + title: Query Classifier API + slug: query-classifier-api + order: 130 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: query_classifier_api.md diff --git a/docs/_src/api/pydoc/question-generator.yml b/docs/_src/api/pydoc/question-generator.yml index 4d5256863..9a066b909 100644 --- a/docs/_src/api/pydoc/question-generator.yml +++ b/docs/_src/api/pydoc/question-generator.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: question_generator.md + type: renderers.ReadmeRenderer + excerpt: Takes a Document as input and generates questions which it believes the Document can answer. + category: 6310ca73c622850ddd3875a2 + title: Question Generator API + slug: question-generator-api + order: 140 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: question_generator_api.md diff --git a/docs/_src/api/pydoc/ranker.yml b/docs/_src/api/pydoc/ranker.yml index dfbce80af..2efd370b3 100644 --- a/docs/_src/api/pydoc/ranker.yml +++ b/docs/_src/api/pydoc/ranker.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: ranker.md + type: renderers.ReadmeRenderer + excerpt: Reorders a set of Documents based on their relevance to the Query. + category: 6310ca73c622850ddd3875a2 + title: Ranker API + slug: ranker-api + order: 150 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: ranker_api.md diff --git a/docs/_src/api/pydoc/reader.yml b/docs/_src/api/pydoc/reader.yml index 1910d36e7..e4f712c9a 100644 --- a/docs/_src/api/pydoc/reader.yml +++ b/docs/_src/api/pydoc/reader.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: reader.md + type: renderers.ReadmeRenderer + excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents. + category: 6310ca73c622850ddd3875a2 + title: Reader API + slug: reader-api + order: 160 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: reader_api.md diff --git a/docs/_src/api/pydoc/retriever.yml b/docs/_src/api/pydoc/retriever.yml index ee64cdfa0..9878f5d28 100644 --- a/docs/_src/api/pydoc/retriever.yml +++ b/docs/_src/api/pydoc/retriever.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: retriever.md + type: renderers.ReadmeRenderer + excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query. + category: 6310ca73c622850ddd3875a2 + title: Retriever API + slug: retriever-api + order: 170 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: retriever_api.md diff --git a/docs/_src/api/pydoc/summarizer.yml b/docs/_src/api/pydoc/summarizer.yml index d6c53bc25..c3918e02f 100644 --- a/docs/_src/api/pydoc/summarizer.yml +++ b/docs/_src/api/pydoc/summarizer.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: summarizer.md + type: renderers.ReadmeRenderer + excerpt: The Summarizer gives a short overview of a long Document. + category: 6310ca73c622850ddd3875a2 + title: Summarizer API + slug: summarizer-api + order: 180 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: summarizer_api.md diff --git a/docs/_src/api/pydoc/translator.yml b/docs/_src/api/pydoc/translator.yml index 36038321e..0cb9eacc4 100644 --- a/docs/_src/api/pydoc/translator.yml +++ b/docs/_src/api/pydoc/translator.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: translator.md + type: renderers.ReadmeRenderer + excerpt: Does what it says on the tin - it translates text from one language into another. + category: 6310ca73c622850ddd3875a2 + title: Translator API + slug: translator-api + order: 190 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: translator_api.md \ No newline at end of file diff --git a/docs/_src/api/pydoc/utils.yml b/docs/_src/api/pydoc/utils.yml index 36c07e190..6782b1529 100644 --- a/docs/_src/api/pydoc/utils.yml +++ b/docs/_src/api/pydoc/utils.yml @@ -12,9 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: markdown - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: utils.md + type: renderers.ReadmeRenderer + excerpt: Utility functions for Haystack. + category: 6310ca73c622850ddd3875a2 + title: Translator API + slug: translator-api + order: 200 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: utils_api.md \ No newline at end of file diff --git a/docs/pydoc/renderers.py b/docs/pydoc/renderers.py index 5c76bdb4a..18be528ef 100644 --- a/docs/pydoc/renderers.py +++ b/docs/pydoc/renderers.py @@ -12,6 +12,9 @@ README_FRONTMATTER = """--- title: {title} excerpt: {excerpt} category: {category} +slug: {slug} +order: {order} +hidden: false --- """ @@ -29,6 +32,8 @@ class ReadmeRenderer(Renderer): title: str category: str excerpt: str + slug: str + order: int # This exposes a special `markdown` settings value that can be used to pass # parameters to the underlying `MarkdownRenderer` markdown: MarkdownRenderer = dataclasses.field(default_factory=MarkdownRenderer) @@ -46,4 +51,6 @@ class ReadmeRenderer(Renderer): self.markdown.render_to_stream(modules, t.cast(t.TextIO, fp)) def _frontmatter(self) -> str: - return README_FRONTMATTER.format(title=self.title, category=self.category, excerpt=self.excerpt) + return README_FRONTMATTER.format( + title=self.title, category=self.category, excerpt=self.excerpt, slug=self.slug, order=self.order + )