docs: sync Haystack API with Readme (#3223)

* First pass at syncing Haystack API with Readme * Reapply changes * Regularize slugs * Regularize slugs * Regularize slugs * Set category id and regen * Trigger workflow * Delete old md files * Test sync * Undo test string * Incorporate reviewer feedback * Test on the fly API generation and sync * Test on the fly API generation and sync * Test on the fly API generation and sync * Test on the fly API generation and sync * Test on the fly API generation and sync * Change name of pydoc-markdown scripts * Test on the fly API generation and sync * Remove version tag * Test version tag * Test version tag * Test version tag * Revert test docstring * Revert md file changes * Revert md file changes * Revert script naming * Test on the fly generation and sync * Adjust for on the fly generation and sync * Revert test string * Remove old documentation workflow * Set workflow to work on main * Change readme version name
2025-08-18 21:49:05 +00:00 · 2022-09-21 17:18:34 +02:00 · 2022-09-21 17:18:34 +02:00 · 492a8046d8
commit 492a8046d8
parent 8f76d64f6f
25 changed files with 301 additions and 178 deletions
--- a/.github/utils/pydoc-markdown.sh
+++ b/.github/utils/pydoc-markdown.sh
@ -1,7 +1,10 @@
 #!/bin/bash

 set -e   # Fails on any error in the following loop
+export PYTHONPATH=$PWD/docs/pydoc # Make the renderers available to pydoc
 cd docs/_src/api/api/
-for file in ../pydoc/* ; do
+mkdir temp
+cd temp
+for file in ../../pydoc/* ; do
    pydoc-markdown "$file"
 done
--- a/.github/workflows/documentation.yml
+++ b/.github/workflows/documentation.yml
@ -1,49 +0,0 @@
-name: Documentation
-
-on:
-  workflow_dispatch: # Activate this workflow manually
-  pull_request:
-
-jobs:
-
-  api-docs:
-    runs-on: ubuntu-latest
-    steps:
-
-      - uses: actions/checkout@v2
-
-      - name: Setup Python
-        uses: ./.github/actions/python_cache/
-
-      - name: Install Haystack
-        run: |
-          pip install --upgrade pip
-          pip install -U .[dev]
-
-      - name: Update API documentation
-        run: .github/utils/pydoc-markdown.sh
-
-      - name: Check status
-        run: |
-          if [[ `git status --porcelain` ]]; then
-            git status
-            echo "###################################################################################################"
-            echo "# "
-            echo "# CHECK FAILED! The API docs were not updated."
-            echo "# "
-            echo "# Either:"
-            echo "# 1. Generate the new API docs locally before committing:"
-            echo "# "
-            echo "#     .github/utils/pydoc-markdown.sh"
-            echo "# "
-            echo "# 2. Install the pre-commit hook:"
-            echo "# "
-            echo "#     pre-commit install --hook-type pre-push"
-            echo "# "
-            echo "# 3. See https://github.com/deepset-ai/haystack/blob/main/CONTRIBUTING.md for help."
-            echo "# "
-            echo "# If you have further problems, please open an issue: https://github.com/deepset-ai/haystack/issues"
-            echo "# "
-            echo "###################################################################################################"
-            exit 1
-          fi
--- a/.github/workflows/readme_integration.yml
+++ b/.github/workflows/readme_integration.yml
@ -0,0 +1,34 @@
+name: Testing Haystack-Readme integration
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout this repo
+        uses: actions/checkout@v3
+
+      - name: Set up Python 3.8.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8.10
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pydoc-markdown==4.5.1
+
+      - name: Generate API docs
+        run: ./.github/utils/pydoc-markdown.sh
+
+      - name: Github Action (API sync)
+        uses: readmeio/rdme@7.2.0
+        env:
+          README_API_KEY: ${{ secrets.README_API_KEY }}
+        with:
+          rdme: docs ./docs/_src/api/api/temp --key="$README_API_KEY" --version=1.10-unstable
--- a/docs/_src/api/pydoc/answer-generator.yml
+++ b/docs/_src/api/pydoc/answer-generator.yml
@ -12,9 +12,17 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: generator.md
+   type: renderers.ReadmeRenderer
+   excerpt: Reads a set of documents and generates an answer to a question, word by word
+   category: 6310ca73c622850ddd3875a2
+   title: Answer Generator API
+   slug: answer-generator-api
+   order: 0
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: answer_generator_api.md
+
+
--- a/docs/_src/api/pydoc/crawler.yml
+++ b/docs/_src/api/pydoc/crawler.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: crawler.md
+   type: renderers.ReadmeRenderer
+   excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file.
+   category: 6310ca73c622850ddd3875a2
+   title: Crawler API
+   slug: crawler-api
+   order: 10
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: crawler_api.md
--- a/docs/_src/api/pydoc/document-classifier.yml
+++ b/docs/_src/api/pydoc/document-classifier.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: document_classifier.md
+   type: renderers.ReadmeRenderer
+   excerpt: Used to create predictions that are attached to documents as metadata.
+   category: 6310ca73c622850ddd3875a2
+   title: Document Classifier API
+   slug: document-classifier-api
+   order: 20
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: document_classifier_api.md
--- a/docs/_src/api/pydoc/document-store.yml
+++ b/docs/_src/api/pydoc/document-store.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: document_store.md
+   type: renderers.ReadmeRenderer
+   excerpt: Stores your texts and meta data and provides them to the Retriever at query time.
+   category: 6310ca73c622850ddd3875a2
+   title: Document Store API
+   slug: document-store-api
+   order: 30
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: document_store_api.md
--- a/docs/_src/api/pydoc/evaluation.yml
+++ b/docs/_src/api/pydoc/evaluation.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: evaluation.md
+   type: renderers.ReadmeRenderer
+   excerpt: The tools needed to evaluate whole pipelines or individual Nodes.
+   category: 6310ca73c622850ddd3875a2
+   title: Evaluation API
+   slug: evaluation-api
+   order: 40
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: evaluation_api.md
--- a/docs/_src/api/pydoc/extractor.yml
+++ b/docs/_src/api/pydoc/extractor.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: extractor.md
+   type: renderers.ReadmeRenderer
+   excerpt: Extracts predefined entities out of a piece of text.
+   category: 6310ca73c622850ddd3875a2
+   title: Entity Extractor API
+   slug: entity-extractor-api
+   order: 50
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: entity_extractor_api.md
--- a/docs/_src/api/pydoc/file-classifier.yml
+++ b/docs/_src/api/pydoc/file-classifier.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: file_classifier.md
+   type: renderers.ReadmeRenderer
+   excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline.
+   category: 6310ca73c622850ddd3875a2
+   title: File Classifier API
+   slug: file-classifier-api
+   order: 60
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: file_classifier_api.md
--- a/docs/_src/api/pydoc/file-converters.yml
+++ b/docs/_src/api/pydoc/file-converters.yml
@ -5,16 +5,22 @@ loaders:
    ignore_when_discovered: ['__init__']
 processors:
  - type: filter
-    expression:
+    expression: 
    documented_only: true
    do_not_filter_modules: false
    skip_empty_modules: true
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: file_converter.md
+   type: renderers.ReadmeRenderer
+   excerpt: Extracts text from files in different formats and cast it into the unified Document format.
+   category: 6310ca73c622850ddd3875a2
+   title: File Converters API
+   slug: file-converters-api
+   order: 70
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: file_converters_api.md
--- a/docs/_src/api/pydoc/other-nodes.yml
+++ b/docs/_src/api/pydoc/other-nodes.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: other_nodes.md
+   type: renderers.ReadmeRenderer
+   excerpt: The utility classes of Haystack.
+   category: 6310ca73c622850ddd3875a2
+   title: Other API
+   slug: other-api
+   order: 80
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: other_api.md
--- a/docs/_src/api/pydoc/pipelines.yml
+++ b/docs/_src/api/pydoc/pipelines.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: pipelines.md
+   type: renderers.ReadmeRenderer
+   excerpt: Arranges nodes in a predefined flow.
+   category: 6310ca73c622850ddd3875a2
+   title: Pipelines API
+   slug: pipelines-api
+   order: 90
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: pipelines_api.md
--- a/docs/_src/api/pydoc/preprocessor.yml
+++ b/docs/_src/api/pydoc/preprocessor.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: preprocessor.md
+   type: renderers.ReadmeRenderer
+   excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces.
+   category: 6310ca73c622850ddd3875a2
+   title: PreProcessor API
+   slug: preprocessor-api
+   order: 100
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: preprocessor_api.md
--- a/docs/_src/api/pydoc/primitives.yml
+++ b/docs/_src/api/pydoc/primitives.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: primitives.md
+   type: renderers.ReadmeRenderer
+   excerpt: These are the core classes that carry data through the system.
+   category: 6310ca73c622850ddd3875a2
+   title: Primitives API
+   slug: primitives-api
+   order: 110
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: primitives_api.md
--- a/docs/_src/api/pydoc/pseudo-label-generator.yml
+++ b/docs/_src/api/pydoc/pseudo-label-generator.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: pseudo_label_generator.md
+   type: renderers.ReadmeRenderer
+   excerpt: Creates training data for dense retrievers without human annotation.
+   category: 6310ca73c622850ddd3875a2
+   title: Pseudo Label Generator API
+   slug: pseudo-label-generator-api
+   order: 120
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: pseudo_label_generator_api.md
--- a/docs/_src/api/pydoc/query-classifier.yml
+++ b/docs/_src/api/pydoc/query-classifier.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: query_classifier.md
+   type: renderers.ReadmeRenderer
+   excerpt: Distinguishes between keyword, question and statements queries.
+   category: 6310ca73c622850ddd3875a2
+   title: Query Classifier API
+   slug: query-classifier-api
+   order: 130
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: query_classifier_api.md
--- a/docs/_src/api/pydoc/question-generator.yml
+++ b/docs/_src/api/pydoc/question-generator.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: question_generator.md
+   type: renderers.ReadmeRenderer
+   excerpt: Takes a Document as input and generates questions which it believes the Document can answer.
+   category: 6310ca73c622850ddd3875a2
+   title: Question Generator API
+   slug: question-generator-api
+   order: 140
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: question_generator_api.md
--- a/docs/_src/api/pydoc/ranker.yml
+++ b/docs/_src/api/pydoc/ranker.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: ranker.md
+   type: renderers.ReadmeRenderer
+   excerpt: Reorders a set of Documents based on their relevance to the Query.
+   category: 6310ca73c622850ddd3875a2
+   title: Ranker API
+   slug: ranker-api
+   order: 150
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: ranker_api.md
--- a/docs/_src/api/pydoc/reader.yml
+++ b/docs/_src/api/pydoc/reader.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: reader.md
+   type: renderers.ReadmeRenderer
+   excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents.
+   category: 6310ca73c622850ddd3875a2
+   title: Reader API
+   slug: reader-api
+   order: 160
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: reader_api.md
--- a/docs/_src/api/pydoc/retriever.yml
+++ b/docs/_src/api/pydoc/retriever.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: retriever.md
+   type: renderers.ReadmeRenderer
+   excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query.
+   category: 6310ca73c622850ddd3875a2
+   title: Retriever API
+   slug: retriever-api
+   order: 170
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: retriever_api.md
--- a/docs/_src/api/pydoc/summarizer.yml
+++ b/docs/_src/api/pydoc/summarizer.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: summarizer.md
+   type: renderers.ReadmeRenderer
+   excerpt: The Summarizer gives a short overview of a long Document.
+   category: 6310ca73c622850ddd3875a2
+   title: Summarizer API
+   slug: summarizer-api
+   order: 180
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: summarizer_api.md
--- a/docs/_src/api/pydoc/translator.yml
+++ b/docs/_src/api/pydoc/translator.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: translator.md
+   type: renderers.ReadmeRenderer
+   excerpt: Does what it says on the tin - it translates text from one language into another.
+   category: 6310ca73c622850ddd3875a2
+   title: Translator API
+   slug: translator-api
+   order: 190
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: translator_api.md
--- a/docs/_src/api/pydoc/utils.yml
+++ b/docs/_src/api/pydoc/utils.yml
@ -12,9 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-  type: markdown
-  descriptive_class_title: false
-  descriptive_module_title: true
-  add_method_class_prefix: true
-  add_member_class_prefix: false
-  filename: utils.md
+   type: renderers.ReadmeRenderer
+   excerpt: Utility functions for Haystack.
+   category: 6310ca73c622850ddd3875a2
+   title: Translator API
+   slug: translator-api
+   order: 200
+   markdown:
+     descriptive_class_title: false
+     descriptive_module_title: true
+     add_method_class_prefix: true
+     add_member_class_prefix: false
+     filename: utils_api.md
--- a/docs/pydoc/renderers.py
+++ b/docs/pydoc/renderers.py
@ -12,6 +12,9 @@ README_FRONTMATTER = """---
 title: {title}
 excerpt: {excerpt}
 category: {category}
+slug: {slug}
+order: {order}
+hidden: false
 ---

 """
@ -29,6 +32,8 @@ class ReadmeRenderer(Renderer):
    title: str
    category: str
    excerpt: str
+    slug: str
+    order: int
    # This exposes a special `markdown` settings value that can be used to pass
    # parameters to the underlying `MarkdownRenderer`
    markdown: MarkdownRenderer = dataclasses.field(default_factory=MarkdownRenderer)
@ -46,4 +51,6 @@ class ReadmeRenderer(Renderer):
                self.markdown.render_to_stream(modules, t.cast(t.TextIO, fp))

    def _frontmatter(self) -> str:
-        return README_FRONTMATTER.format(title=self.title, category=self.category, excerpt=self.excerpt)
+        return README_FRONTMATTER.format(
+            title=self.title, category=self.category, excerpt=self.excerpt, slug=self.slug, order=self.order
+        )