ci: Refactor docs config and generation (#4280)

* Change docs yml category config

* Update docs renderers to fetch categories from Readme.io

* Update readme_sync.yml to handle new docs rendering

* Remove unecessary script and related workflow step

* Fix sys.exits

.github/utils/change_api_category_id.py

@@ -1,55 +0,0 @@
-"""
-Change the category id of the API pydoc configs so that they point to the correct category in Readme.
-"""
-
-import requests
-import pprint
-import base64
-import argparse
-import os
-from pprint import pprint
-
-PYDOC_CONFIGS_DIR = "./docs/_src/api/pydoc"
-
-
-def get_category_id(version):
-    url = "https://dash.readme.com/api/v1/categories/haystack-classes"
-    headers = {"accept": "application/json", "x-readme-version": version, "authorization": api_key_b64}
-    ret = requests.get(url, headers=headers)
-    pprint(ret.text)
-    return ret.json()["id"]
-
-
-def change_api_category_id(new_version, docs_dir):
-    print(new_version)
-    category_id = get_category_id(new_version)
-    print(category_id)
-    ## Replace the category id in the yaml headers
-    for root, dirs, files in os.walk(docs_dir):
-        for file in files:
-            if file.endswith(".yml"):
-                file_path = os.path.join(root, file)
-                lines = [l for l in open(file_path, "r")]
-                for l in lines:
-                    if "category: " in l:
-                        lines[lines.index(l)] = "   category: {}\n".format(category_id)
-                content = "".join(lines)
-                with open(file_path, "w") as f:
-                    f.write(content)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-v", "--version", help="The new minor version that is being released (e.g. v1.9.1).", required=True
-    )
-    parser.add_argument("-k", "--key", help="The Readme API key for Haystack documentation.", required=True)
-    args = parser.parse_args()
-    new_version = args.version
-
-    api_key = args.key
-    api_key += ":"
-    api_key_b64 = "Basic " + base64.b64encode(api_key.encode("utf-8")).decode("utf-8")
-
-    # edit the category id in the yaml headers of pydoc configs
-    change_api_category_id(new_version, PYDOC_CONFIGS_DIR)

.github/workflows/minor_version_release.yml

@@ -48,13 +48,3 @@ jobs:
         run: |
           git checkout main
           python ./.github/utils/release_docs.py --version v${{ steps.versions.outputs.current_release_minor }} --key ${{ secrets.README_API_KEY }}
-
-      - name: Edit category versions in API pydoc configs
-        run: |
-          git checkout v${{ steps.versions.outputs.current_release_minor }}.x
-          python ./.github/utils/change_api_category_id.py --version v${{ steps.versions.outputs.current_release_minor }} --key ${{ secrets.README_API_KEY }}
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git add .
-          git commit -m "Update API docs headers and readme_api_sync.yml to sync to new version"
-          git push

.github/workflows/readme_sync.yml

@@ -23,16 +23,21 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install pydoc-markdown==4.6.4
+          pip install -r docs/pydoc/requirements.txt
 
       - name: Generate API docs
+        env:
+          # This is necessary to fetch the documentation categories
+          # from Readme.io as we need them to associate the slug
+          # in config files with their id.
+          README_API_KEY: ${{ secrets.README_API_KEY }}
         run: ./.github/utils/pydoc-markdown.sh
 
       - name: Get current version
         id: current-version
         shell: bash
         # We only need `major.minor` in Readme so we cut the full version string to the first two tokens
-        run: echo "minor=$(cat VERSION.txt | cut -d "." -f 1,2)" >> $GITHUB_OUTPUT
+        run: echo "minor=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT"
 
       - name: Sync docs with unstable release
         # Instead of putting more logic into the previous step, let's just assume that commits on `main`

docs/pydoc/config/answer-generator.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/answer_generator]
-    modules: ['base', 'transformers', 'openai']
+    modules: ["base", "transformers", "openai"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Reads a set of documents and generates an answer to a question, word by word
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Answer Generator API
   slug: answer-generator-api
   order: 0

docs/pydoc/config/crawler.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/connector]
-    modules: ['crawler']
+    modules: ["crawler"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Crawler API
   slug: crawler-api
   order: 10

docs/pydoc/config/document-classifier.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/document_classifier]
-    modules: ['base', 'transformers']
+    modules: ["base", "transformers"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Used to create predictions that are attached to documents as metadata.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Document Classifier API
   slug: document-classifier-api
   order: 20

docs/pydoc/config/document-store.yml

@@ -1,8 +1,23 @@
 loaders:
   - type: python
     search_path: [../../../haystack/document_stores]
-    modules: ['base', 'elasticsearch', 'opensearch', 'memory', 'sql', 'faiss', 'milvus', 'weaviate', 'graphdb', 'deepsetcloud', 'pinecone', 'search_engine', 'utils']
+    modules:
-    ignore_when_discovered: ['__init__']
+      [
+        "base",
+        "elasticsearch",
+        "opensearch",
+        "memory",
+        "sql",
+        "faiss",
+        "milvus",
+        "weaviate",
+        "graphdb",
+        "deepsetcloud",
+        "pinecone",
+        "search_engine",
+        "utils",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +29,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Stores your texts and meta data and provides them to the Retriever at query time.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Document Store API
   slug: document-store-api
   order: 30

docs/pydoc/config/extractor.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/extractor]
-    modules: ['entity']
+    modules: ["entity"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -16,7 +16,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Extracts predefined entities out of a piece of text.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Entity Extractor API
   slug: entity-extractor-api
   order: 50

docs/pydoc/config/file-classifier.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/file_classifier]
-    modules: ['file_type']
+    modules: ["file_type"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: File Classifier API
   slug: file-classifier-api
   order: 60

docs/pydoc/config/file-converters.yml

@@ -1,8 +1,20 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/file_converter]
-    modules: ['base', 'csv', 'docx', 'image', 'markdown', 'pdf', 'parsr', 'azure', 'tika', 'txt']
+    modules:
-    ignore_when_discovered: ['__init__']
+      [
+        "base",
+        "csv",
+        "docx",
+        "image",
+        "markdown",
+        "pdf",
+        "parsr",
+        "azure",
+        "tika",
+        "txt",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +26,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Extracts text from files in different formats and cast it into the unified Document format.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: File Converters API
   slug: file-converters-api
   order: 70

docs/pydoc/config/other.yml

@@ -1,8 +1,16 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/other]
-    modules: ['docs2answers', 'join_docs', 'join_answers', 'route_documents', 'document_merger', 'shaper']
+    modules:
-    ignore_when_discovered: ['__init__']
+      [
+        "docs2answers",
+        "join_docs",
+        "join_answers",
+        "route_documents",
+        "document_merger",
+        "shaper",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +22,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: The utility classes of Haystack.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Other API
   slug: other-api
   order: 80

docs/pydoc/config/pipelines.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/pipelines]
-    modules: ['base', 'ray', 'standard_pipelines']
+    modules: ["base", "ray", "standard_pipelines"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Arranges nodes in a predefined flow.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Pipelines API
   slug: pipelines-api
   order: 90

docs/pydoc/config/preprocessor.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/preprocessor]
-    modules: ['base', 'preprocessor']
+    modules: ["base", "preprocessor"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: PreProcessor API
   slug: preprocessor-api
   order: 100

docs/pydoc/config/primitives.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack]
-    modules: ['schema']
+    modules: ["schema"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: These are the core classes that carry data through the system.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Primitives API
   slug: primitives-api
   order: 110

docs/pydoc/config/prompt-node.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/prompt]
-    modules: ['prompt_node']
+    modules: ["prompt_node"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -16,7 +16,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Uses Large Language Models directly in your pipelines.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: PromptNode API
   slug: prompt-node-api
   order: 115

docs/pydoc/config/pseudo-label-generator.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/label_generator]
-    modules: ['pseudo_label_generator']
+    modules: ["pseudo_label_generator"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Creates training data for dense retrievers without human annotation.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Pseudo Label Generator API
   slug: pseudo-label-generator-api
   order: 120

docs/pydoc/config/query-classifier.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/query_classifier]
-    modules: ['base', 'sklearn', 'transformers']
+    modules: ["base", "sklearn", "transformers"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Distinguishes between keyword, question and statements queries.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Query Classifier API
   slug: query-classifier-api
   order: 130

docs/pydoc/config/question-generator.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/question_generator]
-    modules: ['question_generator']
+    modules: ["question_generator"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Takes a Document as input and generates questions which it believes the Document can answer.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Question Generator API
   slug: question-generator-api
   order: 140

docs/pydoc/config/ranker.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/ranker]
-    modules: ['base', 'sentence_transformers']
+    modules: ["base", "sentence_transformers"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Reorders a set of Documents based on their relevance to the Query.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Ranker API
   slug: ranker-api
   order: 150

docs/pydoc/config/reader.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/reader]
-    modules: ['base', 'farm', 'transformers', 'table']
+    modules: ["base", "farm", "transformers", "table"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Reader API
   slug: reader-api
   order: 160

docs/pydoc/config/retriever.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/retriever]
-    modules: ['base', 'sparse', 'dense', 'text2sparql', 'multimodal/retriever']
+    modules: ["base", "sparse", "dense", "text2sparql", "multimodal/retriever"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Retriever API
   slug: retriever-api
   order: 170

docs/pydoc/config/summarizer.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/summarizer]
-    modules: ['base', 'transformers']
+    modules: ["base", "transformers"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: The Summarizer gives a short overview of a long Document.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Summarizer API
   slug: summarizer-api
   order: 180

docs/pydoc/config/transformers-img-to-text.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/image_to_text]
-    modules: ['base', 'transformers']
+    modules: ["base", "transformers"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Generates captions for images.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: TransformersImageToText API
   slug: transformers-image-to-text-api
   order: 185

docs/pydoc/config/translator.yml

@@ -1,8 +1,8 @@
 loaders:
   - type: python
     search_path: [../../../haystack/nodes/translator]
-    modules: ['base', 'transformers']
+    modules: ["base", "transformers"]
-    ignore_when_discovered: ['__init__']
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +14,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Does what it says on the tin - it translates text from one language into another.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Translator API
   slug: translator-api
   order: 190

docs/pydoc/config/utils.yml

@@ -1,8 +1,19 @@
 loaders:
   - type: python
     search_path: [../../../haystack/utils]
-    modules: ['doc_store', 'export_utils', 'preprocessing', 'squad_data', 'early_stopping', 'cleaning', 'context_matching', 'deepsetcloud', 'docker']
+    modules:
-    ignore_when_discovered: ['__init__']
+      [
+        "doc_store",
+        "export_utils",
+        "preprocessing",
+        "squad_data",
+        "early_stopping",
+        "cleaning",
+        "context_matching",
+        "deepsetcloud",
+        "docker",
+      ]
+    ignore_when_discovered: ["__init__"]
 processors:
   - type: filter
     expression:
@@ -14,7 +25,7 @@ processors:
 renderer:
   type: renderers.ReadmeRenderer
   excerpt: Utility functions for Haystack.
-   category: 6310ca73c622850ddd3875a2
+  category_slug: haystack-classes
   title: Utils API
   slug: utils-api
   order: 200

docs/pydoc/renderers.py

@@ -1,9 +1,13 @@
+import os
 import sys
 import io
 import dataclasses
-import docspec
 import typing as t
+import base64
 from pathlib import Path
+
+import requests
+import docspec
 from pydoc_markdown.interfaces import Context, Renderer
 from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer
 
@@ -30,16 +34,51 @@ class ReadmeRenderer(Renderer):
 
     # These settings will be used in the front matter output
     title: str
-    category: str
+    category_slug: str
     excerpt: str
     slug: str
     order: int
+    # Docs categories fetched from Readme.io
+    categories: t.Dict[str, str] = dataclasses.field(init=False)
     # This exposes a special `markdown` settings value that can be used to pass
     # parameters to the underlying `MarkdownRenderer`
     markdown: MarkdownRenderer = dataclasses.field(default_factory=MarkdownRenderer)
 
     def init(self, context: Context) -> None:
         self.markdown.init(context)
+        version = self._doc_version()
+        self.categories = self._readme_categories(version)
+
+    def _doc_version(self) -> str:
+        """
+        Returns the docs version.
+        """
+        root = Path(__file__).absolute().parent.parent.parent
+        full_version = (root / "VERSION.txt").read_text()
+        major, minor = full_version.split(".")[:2]
+        if "rc0" in full_version:
+            return f"v{major}.{minor}-unstable"
+        return f"v{major}.{minor}"
+
+    def _readme_categories(self, version: str) -> t.Dict[str, str]:
+        """
+        Fetch the categories of the given version from Readme.io.
+        README_API_KEY env var must be set to correctly get the categories.
+        Returns dictionary containing all the categories slugs and their ids.
+        """
+        README_API_KEY = os.getenv("README_API_KEY")
+        if not README_API_KEY:
+            sys.exit("README_API_KEY env var is not set")
+
+        token = base64.b64encode(f"{README_API_KEY}:".encode()).decode()
+        headers = {"authorization": f"Basic {token}", "x-readme-version": version}
+
+        res = requests.get("https://dash.readme.com/api/v1/categories", headers=headers, timeout=60)
+
+        if not res.ok:
+            sys.exit(f"Error requesting {version} categories")
+
+        return {c["slug"]: c["id"] for c in res.json()}
 
     def render(self, modules: t.List[docspec.Module]) -> None:
         if self.markdown.filename is None:
@@ -52,5 +91,9 @@ class ReadmeRenderer(Renderer):
 
     def _frontmatter(self) -> str:
         return README_FRONTMATTER.format(
-            title=self.title, category=self.category, excerpt=self.excerpt, slug=self.slug, order=self.order
+            title=self.title,
+            category=self.categories[self.category_slug],
+            excerpt=self.excerpt,
+            slug=self.slug,
+            order=self.order,
         )

docs/pydoc/requirements.txt

@@ -0,0 +1,2 @@
+pydoc-markdown==4.6.4
+requests==2.28.2