From ee7442121294e84409d3f4921b541fdd7e2efe3c Mon Sep 17 00:00:00 2001 From: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> Date: Wed, 1 Mar 2023 09:51:02 +0100 Subject: [PATCH] ci: Refactor docs config and generation (#4280) * Change docs yml category config * Update docs renderers to fetch categories from Readme.io * Update readme_sync.yml to handle new docs rendering * Remove unecessary script and related workflow step * Fix sys.exits --- .github/utils/change_api_category_id.py | 55 ------------------- .github/workflows/minor_version_release.yml | 10 ---- .github/workflows/readme_sync.yml | 9 ++- docs/pydoc/config/answer-generator.yml | 28 +++++----- docs/pydoc/config/crawler.yml | 28 +++++----- docs/pydoc/config/document-classifier.yml | 28 +++++----- docs/pydoc/config/document-store.yml | 43 ++++++++++----- docs/pydoc/config/extractor.yml | 28 +++++----- docs/pydoc/config/file-classifier.yml | 28 +++++----- docs/pydoc/config/file-converters.yml | 40 +++++++++----- docs/pydoc/config/other.yml | 36 +++++++----- docs/pydoc/config/pipelines.yml | 28 +++++----- docs/pydoc/config/preprocessor.yml | 28 +++++----- docs/pydoc/config/primitives.yml | 28 +++++----- docs/pydoc/config/prompt-node.yml | 28 +++++----- docs/pydoc/config/pseudo-label-generator.yml | 28 +++++----- docs/pydoc/config/query-classifier.yml | 28 +++++----- docs/pydoc/config/question-generator.yml | 28 +++++----- docs/pydoc/config/ranker.yml | 28 +++++----- docs/pydoc/config/reader.yml | 28 +++++----- docs/pydoc/config/retriever.yml | 28 +++++----- docs/pydoc/config/summarizer.yml | 28 +++++----- .../pydoc/config/transformers-img-to-text.yml | 28 +++++----- docs/pydoc/config/translator.yml | 28 +++++----- docs/pydoc/config/utils.yml | 39 ++++++++----- docs/pydoc/renderers.py | 49 ++++++++++++++++- docs/pydoc/requirements.txt | 2 + 27 files changed, 409 insertions(+), 378 deletions(-) delete mode 100644 .github/utils/change_api_category_id.py create mode 100644 docs/pydoc/requirements.txt diff --git a/.github/utils/change_api_category_id.py b/.github/utils/change_api_category_id.py deleted file mode 100644 index 859828433..000000000 --- a/.github/utils/change_api_category_id.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Change the category id of the API pydoc configs so that they point to the correct category in Readme. -""" - -import requests -import pprint -import base64 -import argparse -import os -from pprint import pprint - -PYDOC_CONFIGS_DIR = "./docs/_src/api/pydoc" - - -def get_category_id(version): - url = "https://dash.readme.com/api/v1/categories/haystack-classes" - headers = {"accept": "application/json", "x-readme-version": version, "authorization": api_key_b64} - ret = requests.get(url, headers=headers) - pprint(ret.text) - return ret.json()["id"] - - -def change_api_category_id(new_version, docs_dir): - print(new_version) - category_id = get_category_id(new_version) - print(category_id) - ## Replace the category id in the yaml headers - for root, dirs, files in os.walk(docs_dir): - for file in files: - if file.endswith(".yml"): - file_path = os.path.join(root, file) - lines = [l for l in open(file_path, "r")] - for l in lines: - if "category: " in l: - lines[lines.index(l)] = " category: {}\n".format(category_id) - content = "".join(lines) - with open(file_path, "w") as f: - f.write(content) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "-v", "--version", help="The new minor version that is being released (e.g. v1.9.1).", required=True - ) - parser.add_argument("-k", "--key", help="The Readme API key for Haystack documentation.", required=True) - args = parser.parse_args() - new_version = args.version - - api_key = args.key - api_key += ":" - api_key_b64 = "Basic " + base64.b64encode(api_key.encode("utf-8")).decode("utf-8") - - # edit the category id in the yaml headers of pydoc configs - change_api_category_id(new_version, PYDOC_CONFIGS_DIR) diff --git a/.github/workflows/minor_version_release.yml b/.github/workflows/minor_version_release.yml index 18390352c..bb3372eb2 100644 --- a/.github/workflows/minor_version_release.yml +++ b/.github/workflows/minor_version_release.yml @@ -48,13 +48,3 @@ jobs: run: | git checkout main python ./.github/utils/release_docs.py --version v${{ steps.versions.outputs.current_release_minor }} --key ${{ secrets.README_API_KEY }} - - - name: Edit category versions in API pydoc configs - run: | - git checkout v${{ steps.versions.outputs.current_release_minor }}.x - python ./.github/utils/change_api_category_id.py --version v${{ steps.versions.outputs.current_release_minor }} --key ${{ secrets.README_API_KEY }} - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - git add . - git commit -m "Update API docs headers and readme_api_sync.yml to sync to new version" - git push diff --git a/.github/workflows/readme_sync.yml b/.github/workflows/readme_sync.yml index fed0416f0..23cebaa2d 100644 --- a/.github/workflows/readme_sync.yml +++ b/.github/workflows/readme_sync.yml @@ -23,16 +23,21 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pydoc-markdown==4.6.4 + pip install -r docs/pydoc/requirements.txt - name: Generate API docs + env: + # This is necessary to fetch the documentation categories + # from Readme.io as we need them to associate the slug + # in config files with their id. + README_API_KEY: ${{ secrets.README_API_KEY }} run: ./.github/utils/pydoc-markdown.sh - name: Get current version id: current-version shell: bash # We only need `major.minor` in Readme so we cut the full version string to the first two tokens - run: echo "minor=$(cat VERSION.txt | cut -d "." -f 1,2)" >> $GITHUB_OUTPUT + run: echo "minor=$(cut -d "." -f 1,2 < VERSION.txt)" >> "$GITHUB_OUTPUT" - name: Sync docs with unstable release # Instead of putting more logic into the previous step, let's just assume that commits on `main` diff --git a/docs/pydoc/config/answer-generator.yml b/docs/pydoc/config/answer-generator.yml index dcfe4be69..a399004b5 100644 --- a/docs/pydoc/config/answer-generator.yml +++ b/docs/pydoc/config/answer-generator.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/answer_generator] - modules: ['base', 'transformers', 'openai'] - ignore_when_discovered: ['__init__'] + modules: ["base", "transformers", "openai"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Reads a set of documents and generates an answer to a question, word by word - category: 6310ca73c622850ddd3875a2 - title: Answer Generator API - slug: answer-generator-api - order: 0 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: answer_generator_api.md + type: renderers.ReadmeRenderer + excerpt: Reads a set of documents and generates an answer to a question, word by word + category_slug: haystack-classes + title: Answer Generator API + slug: answer-generator-api + order: 0 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: answer_generator_api.md diff --git a/docs/pydoc/config/crawler.yml b/docs/pydoc/config/crawler.yml index 78d96ed54..f5d50e76d 100644 --- a/docs/pydoc/config/crawler.yml +++ b/docs/pydoc/config/crawler.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/connector] - modules: ['crawler'] - ignore_when_discovered: ['__init__'] + modules: ["crawler"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file. - category: 6310ca73c622850ddd3875a2 - title: Crawler API - slug: crawler-api - order: 10 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: crawler_api.md + type: renderers.ReadmeRenderer + excerpt: The Crawler scrapes the text from a website, creates a Haystack Document object out of it, and saves it to a JSON file. + category_slug: haystack-classes + title: Crawler API + slug: crawler-api + order: 10 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: crawler_api.md diff --git a/docs/pydoc/config/document-classifier.yml b/docs/pydoc/config/document-classifier.yml index 3899f0578..5fae36149 100644 --- a/docs/pydoc/config/document-classifier.yml +++ b/docs/pydoc/config/document-classifier.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/document_classifier] - modules: ['base', 'transformers'] - ignore_when_discovered: ['__init__'] + modules: ["base", "transformers"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Used to create predictions that are attached to documents as metadata. - category: 6310ca73c622850ddd3875a2 - title: Document Classifier API - slug: document-classifier-api - order: 20 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: document_classifier_api.md + type: renderers.ReadmeRenderer + excerpt: Used to create predictions that are attached to documents as metadata. + category_slug: haystack-classes + title: Document Classifier API + slug: document-classifier-api + order: 20 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: document_classifier_api.md diff --git a/docs/pydoc/config/document-store.yml b/docs/pydoc/config/document-store.yml index c884969ee..d70ca0fd4 100644 --- a/docs/pydoc/config/document-store.yml +++ b/docs/pydoc/config/document-store.yml @@ -1,8 +1,23 @@ loaders: - type: python search_path: [../../../haystack/document_stores] - modules: ['base', 'elasticsearch', 'opensearch', 'memory', 'sql', 'faiss', 'milvus', 'weaviate', 'graphdb', 'deepsetcloud', 'pinecone', 'search_engine', 'utils'] - ignore_when_discovered: ['__init__'] + modules: + [ + "base", + "elasticsearch", + "opensearch", + "memory", + "sql", + "faiss", + "milvus", + "weaviate", + "graphdb", + "deepsetcloud", + "pinecone", + "search_engine", + "utils", + ] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +27,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Stores your texts and meta data and provides them to the Retriever at query time. - category: 6310ca73c622850ddd3875a2 - title: Document Store API - slug: document-store-api - order: 30 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: document_store_api.md + type: renderers.ReadmeRenderer + excerpt: Stores your texts and meta data and provides them to the Retriever at query time. + category_slug: haystack-classes + title: Document Store API + slug: document-store-api + order: 30 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: document_store_api.md diff --git a/docs/pydoc/config/extractor.yml b/docs/pydoc/config/extractor.yml index 69fb04588..29de80b49 100644 --- a/docs/pydoc/config/extractor.yml +++ b/docs/pydoc/config/extractor.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/extractor] - modules: ['entity'] - ignore_when_discovered: ['__init__'] + modules: ["entity"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -14,15 +14,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Extracts predefined entities out of a piece of text. - category: 6310ca73c622850ddd3875a2 - title: Entity Extractor API - slug: entity-extractor-api - order: 50 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: entity_extractor_api.md + type: renderers.ReadmeRenderer + excerpt: Extracts predefined entities out of a piece of text. + category_slug: haystack-classes + title: Entity Extractor API + slug: entity-extractor-api + order: 50 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: entity_extractor_api.md diff --git a/docs/pydoc/config/file-classifier.yml b/docs/pydoc/config/file-classifier.yml index 719405477..d9345e791 100644 --- a/docs/pydoc/config/file-classifier.yml +++ b/docs/pydoc/config/file-classifier.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/file_classifier] - modules: ['file_type'] - ignore_when_discovered: ['__init__'] + modules: ["file_type"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline. - category: 6310ca73c622850ddd3875a2 - title: File Classifier API - slug: file-classifier-api - order: 60 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: file_classifier_api.md + type: renderers.ReadmeRenderer + excerpt: Distinguishes between text, PDF, Markdown, Docx and HTML files and routes them to the appropriate File Converter in an indexing pipeline. + category_slug: haystack-classes + title: File Classifier API + slug: file-classifier-api + order: 60 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: file_classifier_api.md diff --git a/docs/pydoc/config/file-converters.yml b/docs/pydoc/config/file-converters.yml index 419eae003..819ef88ad 100644 --- a/docs/pydoc/config/file-converters.yml +++ b/docs/pydoc/config/file-converters.yml @@ -1,8 +1,20 @@ loaders: - type: python search_path: [../../../haystack/nodes/file_converter] - modules: ['base', 'csv', 'docx', 'image', 'markdown', 'pdf', 'parsr', 'azure', 'tika', 'txt'] - ignore_when_discovered: ['__init__'] + modules: + [ + "base", + "csv", + "docx", + "image", + "markdown", + "pdf", + "parsr", + "azure", + "tika", + "txt", + ] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +24,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Extracts text from files in different formats and cast it into the unified Document format. - category: 6310ca73c622850ddd3875a2 - title: File Converters API - slug: file-converters-api - order: 70 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: file_converters_api.md + type: renderers.ReadmeRenderer + excerpt: Extracts text from files in different formats and cast it into the unified Document format. + category_slug: haystack-classes + title: File Converters API + slug: file-converters-api + order: 70 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: file_converters_api.md diff --git a/docs/pydoc/config/other.yml b/docs/pydoc/config/other.yml index 1959200a6..7458aa9dc 100644 --- a/docs/pydoc/config/other.yml +++ b/docs/pydoc/config/other.yml @@ -1,8 +1,16 @@ loaders: - type: python search_path: [../../../haystack/nodes/other] - modules: ['docs2answers', 'join_docs', 'join_answers', 'route_documents', 'document_merger', 'shaper'] - ignore_when_discovered: ['__init__'] + modules: + [ + "docs2answers", + "join_docs", + "join_answers", + "route_documents", + "document_merger", + "shaper", + ] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +20,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: The utility classes of Haystack. - category: 6310ca73c622850ddd3875a2 - title: Other API - slug: other-api - order: 80 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: other_api.md + type: renderers.ReadmeRenderer + excerpt: The utility classes of Haystack. + category_slug: haystack-classes + title: Other API + slug: other-api + order: 80 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: other_api.md diff --git a/docs/pydoc/config/pipelines.yml b/docs/pydoc/config/pipelines.yml index 5632c49c8..aaf3ea4c1 100644 --- a/docs/pydoc/config/pipelines.yml +++ b/docs/pydoc/config/pipelines.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/pipelines] - modules: ['base', 'ray', 'standard_pipelines'] - ignore_when_discovered: ['__init__'] + modules: ["base", "ray", "standard_pipelines"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Arranges nodes in a predefined flow. - category: 6310ca73c622850ddd3875a2 - title: Pipelines API - slug: pipelines-api - order: 90 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: pipelines_api.md + type: renderers.ReadmeRenderer + excerpt: Arranges nodes in a predefined flow. + category_slug: haystack-classes + title: Pipelines API + slug: pipelines-api + order: 90 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: pipelines_api.md diff --git a/docs/pydoc/config/preprocessor.yml b/docs/pydoc/config/preprocessor.yml index 02fb8e3f3..675d7f03b 100644 --- a/docs/pydoc/config/preprocessor.yml +++ b/docs/pydoc/config/preprocessor.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/preprocessor] - modules: ['base', 'preprocessor'] - ignore_when_discovered: ['__init__'] + modules: ["base", "preprocessor"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces. - category: 6310ca73c622850ddd3875a2 - title: PreProcessor API - slug: preprocessor-api - order: 100 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: preprocessor_api.md + type: renderers.ReadmeRenderer + excerpt: Normalize white spaces, gets rid of headers and footers, cleans empty lines in your Documents, or splits them into smaller pieces. + category_slug: haystack-classes + title: PreProcessor API + slug: preprocessor-api + order: 100 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: preprocessor_api.md diff --git a/docs/pydoc/config/primitives.yml b/docs/pydoc/config/primitives.yml index d80fdc060..d7c439b45 100644 --- a/docs/pydoc/config/primitives.yml +++ b/docs/pydoc/config/primitives.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack] - modules: ['schema'] - ignore_when_discovered: ['__init__'] + modules: ["schema"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: These are the core classes that carry data through the system. - category: 6310ca73c622850ddd3875a2 - title: Primitives API - slug: primitives-api - order: 110 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: primitives_api.md + type: renderers.ReadmeRenderer + excerpt: These are the core classes that carry data through the system. + category_slug: haystack-classes + title: Primitives API + slug: primitives-api + order: 110 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: primitives_api.md diff --git a/docs/pydoc/config/prompt-node.yml b/docs/pydoc/config/prompt-node.yml index fce8c0e20..8797e2d0b 100644 --- a/docs/pydoc/config/prompt-node.yml +++ b/docs/pydoc/config/prompt-node.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/prompt] - modules: ['prompt_node'] - ignore_when_discovered: ['__init__'] + modules: ["prompt_node"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -14,15 +14,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Uses Large Language Models directly in your pipelines. - category: 6310ca73c622850ddd3875a2 - title: PromptNode API - slug: prompt-node-api - order: 115 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: prompt_node_api.md + type: renderers.ReadmeRenderer + excerpt: Uses Large Language Models directly in your pipelines. + category_slug: haystack-classes + title: PromptNode API + slug: prompt-node-api + order: 115 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: prompt_node_api.md diff --git a/docs/pydoc/config/pseudo-label-generator.yml b/docs/pydoc/config/pseudo-label-generator.yml index 2a1cb95ad..6557db49f 100644 --- a/docs/pydoc/config/pseudo-label-generator.yml +++ b/docs/pydoc/config/pseudo-label-generator.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/label_generator] - modules: ['pseudo_label_generator'] - ignore_when_discovered: ['__init__'] + modules: ["pseudo_label_generator"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Creates training data for dense retrievers without human annotation. - category: 6310ca73c622850ddd3875a2 - title: Pseudo Label Generator API - slug: pseudo-label-generator-api - order: 120 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: pseudo_label_generator_api.md + type: renderers.ReadmeRenderer + excerpt: Creates training data for dense retrievers without human annotation. + category_slug: haystack-classes + title: Pseudo Label Generator API + slug: pseudo-label-generator-api + order: 120 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: pseudo_label_generator_api.md diff --git a/docs/pydoc/config/query-classifier.yml b/docs/pydoc/config/query-classifier.yml index 3176b9920..7a52b0956 100644 --- a/docs/pydoc/config/query-classifier.yml +++ b/docs/pydoc/config/query-classifier.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/query_classifier] - modules: ['base', 'sklearn', 'transformers'] - ignore_when_discovered: ['__init__'] + modules: ["base", "sklearn", "transformers"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Distinguishes between keyword, question and statements queries. - category: 6310ca73c622850ddd3875a2 - title: Query Classifier API - slug: query-classifier-api - order: 130 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: query_classifier_api.md + type: renderers.ReadmeRenderer + excerpt: Distinguishes between keyword, question and statements queries. + category_slug: haystack-classes + title: Query Classifier API + slug: query-classifier-api + order: 130 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: query_classifier_api.md diff --git a/docs/pydoc/config/question-generator.yml b/docs/pydoc/config/question-generator.yml index 8cac31b0a..9483250a6 100644 --- a/docs/pydoc/config/question-generator.yml +++ b/docs/pydoc/config/question-generator.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/question_generator] - modules: ['question_generator'] - ignore_when_discovered: ['__init__'] + modules: ["question_generator"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Takes a Document as input and generates questions which it believes the Document can answer. - category: 6310ca73c622850ddd3875a2 - title: Question Generator API - slug: question-generator-api - order: 140 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: question_generator_api.md + type: renderers.ReadmeRenderer + excerpt: Takes a Document as input and generates questions which it believes the Document can answer. + category_slug: haystack-classes + title: Question Generator API + slug: question-generator-api + order: 140 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: question_generator_api.md diff --git a/docs/pydoc/config/ranker.yml b/docs/pydoc/config/ranker.yml index e7f188d7a..b39130bcb 100644 --- a/docs/pydoc/config/ranker.yml +++ b/docs/pydoc/config/ranker.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/ranker] - modules: ['base', 'sentence_transformers'] - ignore_when_discovered: ['__init__'] + modules: ["base", "sentence_transformers"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Reorders a set of Documents based on their relevance to the Query. - category: 6310ca73c622850ddd3875a2 - title: Ranker API - slug: ranker-api - order: 150 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: ranker_api.md + type: renderers.ReadmeRenderer + excerpt: Reorders a set of Documents based on their relevance to the Query. + category_slug: haystack-classes + title: Ranker API + slug: ranker-api + order: 150 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: ranker_api.md diff --git a/docs/pydoc/config/reader.yml b/docs/pydoc/config/reader.yml index 705518f97..161cd58aa 100644 --- a/docs/pydoc/config/reader.yml +++ b/docs/pydoc/config/reader.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/reader] - modules: ['base', 'farm', 'transformers', 'table'] - ignore_when_discovered: ['__init__'] + modules: ["base", "farm", "transformers", "table"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents. - category: 6310ca73c622850ddd3875a2 - title: Reader API - slug: reader-api - order: 160 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: reader_api.md + type: renderers.ReadmeRenderer + excerpt: Takes a question and a set of Documents as input and returns an Answer by selecting a text span within the Documents. + category_slug: haystack-classes + title: Reader API + slug: reader-api + order: 160 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: reader_api.md diff --git a/docs/pydoc/config/retriever.yml b/docs/pydoc/config/retriever.yml index 3bfbb33a7..e09bd0537 100644 --- a/docs/pydoc/config/retriever.yml +++ b/docs/pydoc/config/retriever.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/retriever] - modules: ['base', 'sparse', 'dense', 'text2sparql', 'multimodal/retriever'] - ignore_when_discovered: ['__init__'] + modules: ["base", "sparse", "dense", "text2sparql", "multimodal/retriever"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query. - category: 6310ca73c622850ddd3875a2 - title: Retriever API - slug: retriever-api - order: 170 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: retriever_api.md + type: renderers.ReadmeRenderer + excerpt: Sweeps through a document store and returns a set of candidate documents that are relevant to the query. + category_slug: haystack-classes + title: Retriever API + slug: retriever-api + order: 170 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: retriever_api.md diff --git a/docs/pydoc/config/summarizer.yml b/docs/pydoc/config/summarizer.yml index 4873bdad2..4fcc66655 100644 --- a/docs/pydoc/config/summarizer.yml +++ b/docs/pydoc/config/summarizer.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/summarizer] - modules: ['base', 'transformers'] - ignore_when_discovered: ['__init__'] + modules: ["base", "transformers"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: The Summarizer gives a short overview of a long Document. - category: 6310ca73c622850ddd3875a2 - title: Summarizer API - slug: summarizer-api - order: 180 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: summarizer_api.md + type: renderers.ReadmeRenderer + excerpt: The Summarizer gives a short overview of a long Document. + category_slug: haystack-classes + title: Summarizer API + slug: summarizer-api + order: 180 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: summarizer_api.md diff --git a/docs/pydoc/config/transformers-img-to-text.yml b/docs/pydoc/config/transformers-img-to-text.yml index edfad4a9a..4cb20288e 100644 --- a/docs/pydoc/config/transformers-img-to-text.yml +++ b/docs/pydoc/config/transformers-img-to-text.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/image_to_text] - modules: ['base', 'transformers'] - ignore_when_discovered: ['__init__'] + modules: ["base", "transformers"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Generates captions for images. - category: 6310ca73c622850ddd3875a2 - title: TransformersImageToText API - slug: transformers-image-to-text-api - order: 185 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: transformers-image-to-text.md + type: renderers.ReadmeRenderer + excerpt: Generates captions for images. + category_slug: haystack-classes + title: TransformersImageToText API + slug: transformers-image-to-text-api + order: 185 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: transformers-image-to-text.md diff --git a/docs/pydoc/config/translator.yml b/docs/pydoc/config/translator.yml index 313d5e6bb..d1e230cd7 100644 --- a/docs/pydoc/config/translator.yml +++ b/docs/pydoc/config/translator.yml @@ -1,8 +1,8 @@ loaders: - type: python search_path: [../../../haystack/nodes/translator] - modules: ['base', 'transformers'] - ignore_when_discovered: ['__init__'] + modules: ["base", "transformers"] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +12,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Does what it says on the tin - it translates text from one language into another. - category: 6310ca73c622850ddd3875a2 - title: Translator API - slug: translator-api - order: 190 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: translator_api.md + type: renderers.ReadmeRenderer + excerpt: Does what it says on the tin - it translates text from one language into another. + category_slug: haystack-classes + title: Translator API + slug: translator-api + order: 190 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: translator_api.md diff --git a/docs/pydoc/config/utils.yml b/docs/pydoc/config/utils.yml index 201d135b6..bd7b4dd49 100644 --- a/docs/pydoc/config/utils.yml +++ b/docs/pydoc/config/utils.yml @@ -1,8 +1,19 @@ loaders: - type: python search_path: [../../../haystack/utils] - modules: ['doc_store', 'export_utils', 'preprocessing', 'squad_data', 'early_stopping', 'cleaning', 'context_matching', 'deepsetcloud', 'docker'] - ignore_when_discovered: ['__init__'] + modules: + [ + "doc_store", + "export_utils", + "preprocessing", + "squad_data", + "early_stopping", + "cleaning", + "context_matching", + "deepsetcloud", + "docker", + ] + ignore_when_discovered: ["__init__"] processors: - type: filter expression: @@ -12,15 +23,15 @@ processors: - type: smart - type: crossref renderer: - type: renderers.ReadmeRenderer - excerpt: Utility functions for Haystack. - category: 6310ca73c622850ddd3875a2 - title: Utils API - slug: utils-api - order: 200 - markdown: - descriptive_class_title: false - descriptive_module_title: true - add_method_class_prefix: true - add_member_class_prefix: false - filename: utils_api.md + type: renderers.ReadmeRenderer + excerpt: Utility functions for Haystack. + category_slug: haystack-classes + title: Utils API + slug: utils-api + order: 200 + markdown: + descriptive_class_title: false + descriptive_module_title: true + add_method_class_prefix: true + add_member_class_prefix: false + filename: utils_api.md diff --git a/docs/pydoc/renderers.py b/docs/pydoc/renderers.py index 2e89107e8..14f6c376a 100644 --- a/docs/pydoc/renderers.py +++ b/docs/pydoc/renderers.py @@ -1,9 +1,13 @@ +import os import sys import io import dataclasses -import docspec import typing as t +import base64 from pathlib import Path + +import requests +import docspec from pydoc_markdown.interfaces import Context, Renderer from pydoc_markdown.contrib.renderers.markdown import MarkdownRenderer @@ -30,16 +34,51 @@ class ReadmeRenderer(Renderer): # These settings will be used in the front matter output title: str - category: str + category_slug: str excerpt: str slug: str order: int + # Docs categories fetched from Readme.io + categories: t.Dict[str, str] = dataclasses.field(init=False) # This exposes a special `markdown` settings value that can be used to pass # parameters to the underlying `MarkdownRenderer` markdown: MarkdownRenderer = dataclasses.field(default_factory=MarkdownRenderer) def init(self, context: Context) -> None: self.markdown.init(context) + version = self._doc_version() + self.categories = self._readme_categories(version) + + def _doc_version(self) -> str: + """ + Returns the docs version. + """ + root = Path(__file__).absolute().parent.parent.parent + full_version = (root / "VERSION.txt").read_text() + major, minor = full_version.split(".")[:2] + if "rc0" in full_version: + return f"v{major}.{minor}-unstable" + return f"v{major}.{minor}" + + def _readme_categories(self, version: str) -> t.Dict[str, str]: + """ + Fetch the categories of the given version from Readme.io. + README_API_KEY env var must be set to correctly get the categories. + Returns dictionary containing all the categories slugs and their ids. + """ + README_API_KEY = os.getenv("README_API_KEY") + if not README_API_KEY: + sys.exit("README_API_KEY env var is not set") + + token = base64.b64encode(f"{README_API_KEY}:".encode()).decode() + headers = {"authorization": f"Basic {token}", "x-readme-version": version} + + res = requests.get("https://dash.readme.com/api/v1/categories", headers=headers, timeout=60) + + if not res.ok: + sys.exit(f"Error requesting {version} categories") + + return {c["slug"]: c["id"] for c in res.json()} def render(self, modules: t.List[docspec.Module]) -> None: if self.markdown.filename is None: @@ -52,5 +91,9 @@ class ReadmeRenderer(Renderer): def _frontmatter(self) -> str: return README_FRONTMATTER.format( - title=self.title, category=self.category, excerpt=self.excerpt, slug=self.slug, order=self.order + title=self.title, + category=self.categories[self.category_slug], + excerpt=self.excerpt, + slug=self.slug, + order=self.order, ) diff --git a/docs/pydoc/requirements.txt b/docs/pydoc/requirements.txt new file mode 100644 index 000000000..4887edfaa --- /dev/null +++ b/docs/pydoc/requirements.txt @@ -0,0 +1,2 @@ +pydoc-markdown==4.6.4 +requests==2.28.2