ci: Refactor docs config and generation (#4280)

* Change docs yml category config * Update docs renderers to fetch categories from Readme.io * Update readme_sync.yml to handle new docs rendering * Remove unecessary script and related workflow step * Fix sys.exits
2025-12-05 11:27:15 +00:00 · 2023-03-01 09:51:02 +01:00 · 2023-03-01 09:51:02 +01:00 · ee74421212
commit ee74421212
parent 6e241262ad
27 changed files with 409 additions and 378 deletions
--- a/.github/utils/change_api_category_id.py
+++ b/.github/utils/change_api_category_id.py
@ -1,55 +0,0 @@
-"""
-Change the category id of the API pydoc configs so that they point to the correct category in Readme.
-"""
-
-import requests
-import pprint
-import base64
-import argparse
-import os
-from pprint import pprint
-
-PYDOC_CONFIGS_DIR = "./docs/_src/api/pydoc"
-
-
-def get_category_id(version):
-    url = "https://dash.readme.com/api/v1/categories/haystack-classes"
-    headers = {"accept": "application/json", "x-readme-version": version, "authorization": api_key_b64}
-    ret = requests.get(url, headers=headers)
-    pprint(ret.text)
-    return ret.json()["id"]
-
-
-def change_api_category_id(new_version, docs_dir):
-    print(new_version)
-    category_id = get_category_id(new_version)
-    print(category_id)
-    ## Replace the category id in the yaml headers
-    for root, dirs, files in os.walk(docs_dir):
-        for file in files:
-            if file.endswith(".yml"):
-                file_path = os.path.join(root, file)
-                lines = [l for l in open(file_path, "r")]
-                for l in lines:
-                    if "category: " in l:
-                        lines[lines.index(l)] = "   category: {}\n".format(category_id)
-                content = "".join(lines)
-                with open(file_path, "w") as f:
-                    f.write(content)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-v", "--version", help="The new minor version that is being released (e.g. v1.9.1).", required=True
-    )
-    parser.add_argument("-k", "--key", help="The Readme API key for Haystack documentation.", required=True)
-    args = parser.parse_args()
-    new_version = args.version
-
-    api_key = args.key
-    api_key += ":"
-    api_key_b64 = "Basic " + base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
-
-    # edit the category id in the yaml headers of pydoc configs
-    change_api_category_id(new_version, PYDOC_CONFIGS_DIR)
--- a/.github/workflows/minor_version_release.yml
+++ b/.github/workflows/minor_version_release.yml
@ -48,13 +48,3 @@ jobs:
        run: |
          git checkout main
          python ./.github/utils/release_docs.py --version v${{ steps.versions.outputs.current_release_minor }} --key ${{ secrets.README_API_KEY }}
-
-      - name: Edit category versions in API pydoc configs
-        run: |
-          git checkout v${{ steps.versions.outputs.current_release_minor }}.x
-          python ./.github/utils/change_api_category_id.py --version v${{ steps.versions.outputs.current_release_minor }} --key ${{ secrets.README_API_KEY }}
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git add .
-          git commit -m "Update API docs headers and readme_api_sync.yml to sync to new version"
-          git push
--- a/.github/workflows/readme_sync.yml
+++ b/.github/workflows/readme_sync.yml
@ -23,16 +23,21 @@ jobs:
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
-          pip install pydoc-markdown==4.6.4
+          pip install -r docs/pydoc/requirements.txt

      - name: Generate API docs
+        env:
+          # This is necessary to fetch the documentation categories
+          # from Readme.io as we need them to associate the slug
+          # in config files with their id.
+          README_API_KEY: ${{ secrets.README_API_KEY }}
        run: ./.github/utils/pydoc-markdown.sh

      - name: Get current version
        id: current-version
        shell: bash
        # We only need `major.minor` in Readme so we cut the full version string to the first two tokens
-        run: echo "minor=$(cat VERSION.txt | cut -d "." -f 1,2)" >> $GITHUB_OUTPUT
+        run: echo "minor=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT"

      - name: Sync docs with unstable release
        # Instead of putting more logic into the previous step, let's just assume that commits on `main`
--- a/docs/pydoc/config/answer-generator.yml
+++ b/docs/pydoc/config/answer-generator.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/answer_generator]
-    modules: ['base', 'transformers', 'openai']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "transformers", "openai"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Reads a set of documents and generates an answer to a question, word by word
-   category: 6310ca73c622850ddd3875a2
-   title: Answer Generator API
-   slug: answer-generator-api
-   order: 0
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: answer_generator_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Reads a set of documents and generates an answer to a question, word by word
+  category_slug: haystack-classes
+  title: Answer Generator API
+  slug: answer-generator-api
+  order: 0
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: answer_generator_api.md
--- a/docs/pydoc/config/crawler.yml
+++ b/docs/pydoc/config/crawler.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/connector]
-    modules: ['crawler']
-    ignore_when_discovered: ['__init__']
+    modules: ["crawler"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file.
-   category: 6310ca73c622850ddd3875a2
-   title: Crawler API
-   slug: crawler-api
-   order: 10
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: crawler_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file.
+  category_slug: haystack-classes
+  title: Crawler API
+  slug: crawler-api
+  order: 10
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: crawler_api.md
--- a/docs/pydoc/config/document-classifier.yml
+++ b/docs/pydoc/config/document-classifier.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/document_classifier]
-    modules: ['base', 'transformers']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "transformers"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Used to create predictions that are attached to documents as metadata.
-   category: 6310ca73c622850ddd3875a2
-   title: Document Classifier API
-   slug: document-classifier-api
-   order: 20
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: document_classifier_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Used to create predictions that are attached to documents as metadata.
+  category_slug: haystack-classes
+  title: Document Classifier API
+  slug: document-classifier-api
+  order: 20
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: document_classifier_api.md
--- a/docs/pydoc/config/document-store.yml
+++ b/docs/pydoc/config/document-store.yml
@ -1,8 +1,23 @@
 loaders:
  - type: python
    search_path: [../../../haystack/document_stores]
-    modules: ['base', 'elasticsearch', 'opensearch', 'memory', 'sql', 'faiss', 'milvus', 'weaviate', 'graphdb', 'deepsetcloud', 'pinecone', 'search_engine', 'utils']
-    ignore_when_discovered: ['__init__']
+    modules:
+      [
+        "base",
+        "elasticsearch",
+        "opensearch",
+        "memory",
+        "sql",
+        "faiss",
+        "milvus",
+        "weaviate",
+        "graphdb",
+        "deepsetcloud",
+        "pinecone",
+        "search_engine",
+        "utils",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +27,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Stores your texts and meta data and provides them to the Retriever at query time.
-   category: 6310ca73c622850ddd3875a2
-   title: Document Store API
-   slug: document-store-api
-   order: 30
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: document_store_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Stores your texts and meta data and provides them to the Retriever at query time.
+  category_slug: haystack-classes
+  title: Document Store API
+  slug: document-store-api
+  order: 30
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: document_store_api.md
--- a/docs/pydoc/config/extractor.yml
+++ b/docs/pydoc/config/extractor.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/extractor]
-    modules: ['entity']
-    ignore_when_discovered: ['__init__']
+    modules: ["entity"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -14,15 +14,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Extracts predefined entities out of a piece of text.
-   category: 6310ca73c622850ddd3875a2
-   title: Entity Extractor API
-   slug: entity-extractor-api
-   order: 50
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: entity_extractor_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Extracts predefined entities out of a piece of text.
+  category_slug: haystack-classes
+  title: Entity Extractor API
+  slug: entity-extractor-api
+  order: 50
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: entity_extractor_api.md
--- a/docs/pydoc/config/file-classifier.yml
+++ b/docs/pydoc/config/file-classifier.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/file_classifier]
-    modules: ['file_type']
-    ignore_when_discovered: ['__init__']
+    modules: ["file_type"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline.
-   category: 6310ca73c622850ddd3875a2
-   title: File Classifier API
-   slug: file-classifier-api
-   order: 60
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: file_classifier_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline.
+  category_slug: haystack-classes
+  title: File Classifier API
+  slug: file-classifier-api
+  order: 60
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: file_classifier_api.md
--- a/docs/pydoc/config/file-converters.yml
+++ b/docs/pydoc/config/file-converters.yml
@ -1,8 +1,20 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/file_converter]
-    modules: ['base', 'csv', 'docx', 'image', 'markdown', 'pdf', 'parsr', 'azure', 'tika', 'txt']
-    ignore_when_discovered: ['__init__']
+    modules:
+      [
+        "base",
+        "csv",
+        "docx",
+        "image",
+        "markdown",
+        "pdf",
+        "parsr",
+        "azure",
+        "tika",
+        "txt",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +24,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Extracts text from files in different formats and cast it into the unified Document format.
-   category: 6310ca73c622850ddd3875a2
-   title: File Converters API
-   slug: file-converters-api
-   order: 70
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: file_converters_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Extracts text from files in different formats and cast it into the unified Document format.
+  category_slug: haystack-classes
+  title: File Converters API
+  slug: file-converters-api
+  order: 70
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: file_converters_api.md
--- a/docs/pydoc/config/other.yml
+++ b/docs/pydoc/config/other.yml
@ -1,8 +1,16 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/other]
-    modules: ['docs2answers', 'join_docs', 'join_answers', 'route_documents', 'document_merger', 'shaper']
-    ignore_when_discovered: ['__init__']
+    modules:
+      [
+        "docs2answers",
+        "join_docs",
+        "join_answers",
+        "route_documents",
+        "document_merger",
+        "shaper",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +20,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: The utility classes of Haystack.
-   category: 6310ca73c622850ddd3875a2
-   title: Other API
-   slug: other-api
-   order: 80
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: other_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: The utility classes of Haystack.
+  category_slug: haystack-classes
+  title: Other API
+  slug: other-api
+  order: 80
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: other_api.md
--- a/docs/pydoc/config/pipelines.yml
+++ b/docs/pydoc/config/pipelines.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/pipelines]
-    modules: ['base', 'ray', 'standard_pipelines']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "ray", "standard_pipelines"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Arranges nodes in a predefined flow.
-   category: 6310ca73c622850ddd3875a2
-   title: Pipelines API
-   slug: pipelines-api
-   order: 90
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: pipelines_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Arranges nodes in a predefined flow.
+  category_slug: haystack-classes
+  title: Pipelines API
+  slug: pipelines-api
+  order: 90
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: pipelines_api.md
--- a/docs/pydoc/config/preprocessor.yml
+++ b/docs/pydoc/config/preprocessor.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/preprocessor]
-    modules: ['base', 'preprocessor']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "preprocessor"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces.
-   category: 6310ca73c622850ddd3875a2
-   title: PreProcessor API
-   slug: preprocessor-api
-   order: 100
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: preprocessor_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces.
+  category_slug: haystack-classes
+  title: PreProcessor API
+  slug: preprocessor-api
+  order: 100
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: preprocessor_api.md
--- a/docs/pydoc/config/primitives.yml
+++ b/docs/pydoc/config/primitives.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack]
-    modules: ['schema']
-    ignore_when_discovered: ['__init__']
+    modules: ["schema"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: These are the core classes that carry data through the system.
-   category: 6310ca73c622850ddd3875a2
-   title: Primitives API
-   slug: primitives-api
-   order: 110
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: primitives_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: These are the core classes that carry data through the system.
+  category_slug: haystack-classes
+  title: Primitives API
+  slug: primitives-api
+  order: 110
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: primitives_api.md
--- a/docs/pydoc/config/prompt-node.yml
+++ b/docs/pydoc/config/prompt-node.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/prompt]
-    modules: ['prompt_node']
-    ignore_when_discovered: ['__init__']
+    modules: ["prompt_node"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -14,15 +14,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Uses Large Language Models directly in your pipelines.
-   category: 6310ca73c622850ddd3875a2
-   title: PromptNode API
-   slug: prompt-node-api
-   order: 115
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: prompt_node_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Uses Large Language Models directly in your pipelines.
+  category_slug: haystack-classes
+  title: PromptNode API
+  slug: prompt-node-api
+  order: 115
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: prompt_node_api.md
--- a/docs/pydoc/config/pseudo-label-generator.yml
+++ b/docs/pydoc/config/pseudo-label-generator.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/label_generator]
-    modules: ['pseudo_label_generator']
-    ignore_when_discovered: ['__init__']
+    modules: ["pseudo_label_generator"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Creates training data for dense retrievers without human annotation.
-   category: 6310ca73c622850ddd3875a2
-   title: Pseudo Label Generator API
-   slug: pseudo-label-generator-api
-   order: 120
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: pseudo_label_generator_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Creates training data for dense retrievers without human annotation.
+  category_slug: haystack-classes
+  title: Pseudo Label Generator API
+  slug: pseudo-label-generator-api
+  order: 120
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: pseudo_label_generator_api.md
--- a/docs/pydoc/config/query-classifier.yml
+++ b/docs/pydoc/config/query-classifier.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/query_classifier]
-    modules: ['base', 'sklearn', 'transformers']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "sklearn", "transformers"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Distinguishes between keyword, question and statements queries.
-   category: 6310ca73c622850ddd3875a2
-   title: Query Classifier API
-   slug: query-classifier-api
-   order: 130
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: query_classifier_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Distinguishes between keyword, question and statements queries.
+  category_slug: haystack-classes
+  title: Query Classifier API
+  slug: query-classifier-api
+  order: 130
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: query_classifier_api.md
--- a/docs/pydoc/config/question-generator.yml
+++ b/docs/pydoc/config/question-generator.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/question_generator]
-    modules: ['question_generator']
-    ignore_when_discovered: ['__init__']
+    modules: ["question_generator"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Takes a Document as input and generates questions which it believes the Document can answer.
-   category: 6310ca73c622850ddd3875a2
-   title: Question Generator API
-   slug: question-generator-api
-   order: 140
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: question_generator_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Takes a Document as input and generates questions which it believes the Document can answer.
+  category_slug: haystack-classes
+  title: Question Generator API
+  slug: question-generator-api
+  order: 140
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: question_generator_api.md
--- a/docs/pydoc/config/ranker.yml
+++ b/docs/pydoc/config/ranker.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/ranker]
-    modules: ['base', 'sentence_transformers']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "sentence_transformers"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Reorders a set of Documents based on their relevance to the Query.
-   category: 6310ca73c622850ddd3875a2
-   title: Ranker API
-   slug: ranker-api
-   order: 150
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: ranker_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Reorders a set of Documents based on their relevance to the Query.
+  category_slug: haystack-classes
+  title: Ranker API
+  slug: ranker-api
+  order: 150
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: ranker_api.md
--- a/docs/pydoc/config/reader.yml
+++ b/docs/pydoc/config/reader.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/reader]
-    modules: ['base', 'farm', 'transformers', 'table']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "farm", "transformers", "table"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents.
-   category: 6310ca73c622850ddd3875a2
-   title: Reader API
-   slug: reader-api
-   order: 160
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: reader_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents.
+  category_slug: haystack-classes
+  title: Reader API
+  slug: reader-api
+  order: 160
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: reader_api.md
--- a/docs/pydoc/config/retriever.yml
+++ b/docs/pydoc/config/retriever.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/retriever]
-    modules: ['base', 'sparse', 'dense', 'text2sparql', 'multimodal/retriever']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "sparse", "dense", "text2sparql", "multimodal/retriever"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query.
-   category: 6310ca73c622850ddd3875a2
-   title: Retriever API
-   slug: retriever-api
-   order: 170
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: retriever_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query.
+  category_slug: haystack-classes
+  title: Retriever API
+  slug: retriever-api
+  order: 170
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: retriever_api.md
--- a/docs/pydoc/config/summarizer.yml
+++ b/docs/pydoc/config/summarizer.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/summarizer]
-    modules: ['base', 'transformers']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "transformers"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: The Summarizer gives a short overview of a long Document.
-   category: 6310ca73c622850ddd3875a2
-   title: Summarizer API
-   slug: summarizer-api
-   order: 180
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: summarizer_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: The Summarizer gives a short overview of a long Document.
+  category_slug: haystack-classes
+  title: Summarizer API
+  slug: summarizer-api
+  order: 180
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: summarizer_api.md
--- a/docs/pydoc/config/transformers-img-to-text.yml
+++ b/docs/pydoc/config/transformers-img-to-text.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/image_to_text]
-    modules: ['base', 'transformers']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "transformers"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Generates captions for images.
-   category: 6310ca73c622850ddd3875a2
-   title: TransformersImageToText API
-   slug: transformers-image-to-text-api
-   order: 185
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: transformers-image-to-text.md
+  type: renderers.ReadmeRenderer
+  excerpt: Generates captions for images.
+  category_slug: haystack-classes
+  title: TransformersImageToText API
+  slug: transformers-image-to-text-api
+  order: 185
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: transformers-image-to-text.md
--- a/docs/pydoc/config/translator.yml
+++ b/docs/pydoc/config/translator.yml
@ -1,8 +1,8 @@
 loaders:
  - type: python
    search_path: [../../../haystack/nodes/translator]
-    modules: ['base', 'transformers']
-    ignore_when_discovered: ['__init__']
+    modules: ["base", "transformers"]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +12,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Does what it says on the tin - it translates text from one language into another.
-   category: 6310ca73c622850ddd3875a2
-   title: Translator API
-   slug: translator-api
-   order: 190
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: translator_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Does what it says on the tin - it translates text from one language into another.
+  category_slug: haystack-classes
+  title: Translator API
+  slug: translator-api
+  order: 190
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: translator_api.md
--- a/docs/pydoc/config/utils.yml
+++ b/docs/pydoc/config/utils.yml
@ -1,8 +1,19 @@
 loaders:
  - type: python
    search_path: [../../../haystack/utils]
-    modules: ['doc_store', 'export_utils', 'preprocessing', 'squad_data', 'early_stopping', 'cleaning', 'context_matching', 'deepsetcloud', 'docker']
-    ignore_when_discovered: ['__init__']
+    modules:
+      [
+        "doc_store",
+        "export_utils",
+        "preprocessing",
+        "squad_data",
+        "early_stopping",
+        "cleaning",
+        "context_matching",
+        "deepsetcloud",
+        "docker",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
  - type: filter
    expression:
@ -12,15 +23,15 @@ processors:
  - type: smart
  - type: crossref
 renderer:
-   type: renderers.ReadmeRenderer
-   excerpt: Utility functions for Haystack.
-   category: 6310ca73c622850ddd3875a2
-   title: Utils API
-   slug: utils-api
-   order: 200
-   markdown:
-     descriptive_class_title: false
-     descriptive_module_title: true
-     add_method_class_prefix: true
-     add_member_class_prefix: false
-     filename: utils_api.md
+  type: renderers.ReadmeRenderer
+  excerpt: Utility functions for Haystack.
+  category_slug: haystack-classes
+  title: Utils API
+  slug: utils-api
+  order: 200
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: utils_api.md
--- a/docs/pydoc/renderers.py
+++ b/docs/pydoc/renderers.py
@ -1,9 +1,13 @@
+import os
 import sys
 import io
 import dataclasses
-import docspec
 import typing as t
+import base64
 from pathlib import Path
+
+import requests
+import docspec
 from pydoc_markdown.interfaces import Context, Renderer
 from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer

@ -30,16 +34,51 @@ class ReadmeRenderer(Renderer):

    # These settings will be used in the front matter output
    title: str
-    category: str
+    category_slug: str
    excerpt: str
    slug: str
    order: int
+    # Docs categories fetched from Readme.io
+    categories: t.Dict[str, str] = dataclasses.field(init=False)
    # This exposes a special `markdown` settings value that can be used to pass
    # parameters to the underlying `MarkdownRenderer`
    markdown: MarkdownRenderer = dataclasses.field(default_factory=MarkdownRenderer)

    def init(self, context: Context) -> None:
        self.markdown.init(context)
+        version = self._doc_version()
+        self.categories = self._readme_categories(version)
+
+    def _doc_version(self) -> str:
+        """
+        Returns the docs version.
+        """
+        root = Path(__file__).absolute().parent.parent.parent
+        full_version = (root / "VERSION.txt").read_text()
+        major, minor = full_version.split(".")[:2]
+        if "rc0" in full_version:
+            return f"v{major}.{minor}-unstable"
+        return f"v{major}.{minor}"
+
+    def _readme_categories(self, version: str) -> t.Dict[str, str]:
+        """
+        Fetch the categories of the given version from Readme.io.
+        README_API_KEY env var must be set to correctly get the categories.
+        Returns dictionary containing all the categories slugs and their ids.
+        """
+        README_API_KEY = os.getenv("README_API_KEY")
+        if not README_API_KEY:
+            sys.exit("README_API_KEY env var is not set")
+
+        token = base64.b64encode(f"{README_API_KEY}:".encode()).decode()
+        headers = {"authorization": f"Basic {token}", "x-readme-version": version}
+
+        res = requests.get("https://dash.readme.com/api/v1/categories", headers=headers, timeout=60)
+
+        if not res.ok:
+            sys.exit(f"Error requesting {version} categories")
+
+        return {c["slug"]: c["id"] for c in res.json()}

    def render(self, modules: t.List[docspec.Module]) -> None:
        if self.markdown.filename is None:
@ -52,5 +91,9 @@ class ReadmeRenderer(Renderer):

    def _frontmatter(self) -> str:
        return README_FRONTMATTER.format(
-            title=self.title, category=self.category, excerpt=self.excerpt, slug=self.slug, order=self.order
+            title=self.title,
+            category=self.categories[self.category_slug],
+            excerpt=self.excerpt,
+            slug=self.slug,
+            order=self.order,
        )
--- a/docs/pydoc/requirements.txt
+++ b/docs/pydoc/requirements.txt
@ -0,0 +1,2 @@
+pydoc-markdown==4.6.4
+requests==2.28.2