yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-12-25 14:08:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

docs: Add inherited methods to API reference documentation (#5273)

Browse Source 
* Add inherited methods to API reference documentation

* Fix typing
This commit is contained in:
bogdankostic 2023-08-02 18:54:15 +02:00 committed by GitHub
parent 1876c41f07
commit a26859f065
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 89 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/pydoc/config/agent.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/agents]
modules: ['base', 'conversational', 'utils', 'agent_step', 'memory/conversation_memory', 'memory/conversation_summary_memory']
ignore_when_discovered: ['__init__']

2
docs/pydoc/config/answer-generator.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/answer_generator]
modules: ["base", "openai"]
ignore_when_discovered: ["__init__"]

3
docs/pydoc/config/crawler.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/connector]
modules: ["crawler"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: crawler_api.md

2
docs/pydoc/config/doc-language-classifier.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/doc_language_classifier]
modules: ["base", "langdetect", "transformers"]
ignore_when_discovered: ["__init__"]

3
docs/pydoc/config/document-classifier.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/document_classifier]
modules: ["base", "transformers"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: document_classifier_api.md

2
docs/pydoc/config/document-store.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/document_stores, ../../../haystack/document_stores/elasticsearch]
modules:
[

3
docs/pydoc/config/extractor.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/extractor]
modules: ["entity"]
ignore_when_discovered: ["__init__"]
@ -26,4 +26,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: entity_extractor_api.md

3
docs/pydoc/config/file-classifier.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/file_classifier]
modules: ["file_type"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: file_classifier_api.md

2
docs/pydoc/config/file-converters.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/file_converter]
modules:
[

3
docs/pydoc/config/other.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/other]
modules:
[
@ -32,4 +32,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: other_api.md

2
docs/pydoc/config/pipelines.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/pipelines]
modules: ["base", "ray", "standard_pipelines"]
ignore_when_discovered: ["__init__"]

3
docs/pydoc/config/preprocessor.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/preprocessor]
modules: ["base", "preprocessor"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: preprocessor_api.md

3
docs/pydoc/config/primitives.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack]
modules: ["schema"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: primitives_api.md

2
docs/pydoc/config/prompt-node.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/prompt]
modules:
[

3
docs/pydoc/config/pseudo-label-generator.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/label_generator]
modules: ["pseudo_label_generator"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: pseudo_label_generator_api.md

3
docs/pydoc/config/query-classifier.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/query_classifier]
modules: ["base", "sklearn", "transformers"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: query_classifier_api.md

3
docs/pydoc/config/question-generator.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/question_generator]
modules: ["question_generator"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: question_generator_api.md

2
docs/pydoc/config/ranker.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/ranker]
modules: ["base", "sentence_transformers", "recentness_ranker", "diversity", "lost_in_the_middle"]
ignore_when_discovered: ["__init__"]

3
docs/pydoc/config/reader.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/reader]
modules: ["base", "farm", "transformers", "table"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: reader_api.md

2
docs/pydoc/config/retriever.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/retriever]
modules: ["base", "sparse", "dense", "multimodal/retriever", "web", "link_content"]
ignore_when_discovered: ["__init__"]

2
docs/pydoc/config/shaper.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/other]
modules: ['shaper']
ignore_when_discovered: ['__init__']

2
docs/pydoc/config/summarizer.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/summarizer]
modules: ["base", "transformers"]
ignore_when_discovered: ["__init__"]

3
docs/pydoc/config/transformers-img-to-text.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/image_to_text]
modules: ["base", "transformers"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: transformers-image-to-text.md

3
docs/pydoc/config/translator.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/translator]
modules: ["base", "transformers"]
ignore_when_discovered: ["__init__"]
@ -24,4 +24,3 @@ renderer:
add_method_class_prefix: true
add_member_class_prefix: false
filename: translator_api.md

2
docs/pydoc/config/utils.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/utils]
modules:
[

2
docs/pydoc/config/whisper.yml
View File

@ -1,5 +1,5 @@
loaders:
- type: python
- type: loaders.CustomPythonLoader
search_path: [../../../haystack/nodes/audio]
modules: ["whisper_transcriber"]
ignore_when_discovered: ["__init__"]

63
docs/pydoc/loaders.py Normal file
View File

@ -0,0 +1,63 @@
import copy
import typing as t
import docspec
from pydoc_markdown.interfaces import Context
from pydoc_markdown.contrib.loaders.python import PythonLoader
class CustomPythonLoader(PythonLoader):
def load(self) -> t.Iterable[docspec.Module]:
"""
Load the modules, but include inherited methods in the classes.
"""
# Load all haystack modules
temp_loader = PythonLoader(search_path=["../../../haystack"])
temp_loader.init(Context(directory="."))
all_modules = list(temp_loader.load())
# Collect all classes
classes = {}
for module in all_modules:
for member in module.members:
if isinstance(member, docspec.Class):
classes[member.name] = member
# Load the modules specified in the search path
modules = super().load()
# Add inherited methods to the classes
modules = self.include_inherited_methods(modules, classes)
return modules
def include_inherited_methods(
self, modules: t.Iterable[docspec.Module], classes: t.Dict[str, docspec.Class]
) -> t.Iterable[docspec.Module]:
"""
Recursively include inherited methods from the base classes.
"""
modules = list(modules)
for module in modules:
for cls in module.members:
if isinstance(cls, docspec.Class):
self.include_methods_for_class(cls, classes)
return modules
def include_methods_for_class(self, cls: docspec.Class, classes: t.Dict[str, docspec.Class]):
"""
Include all methods inherited from base classes to the class.
"""
if cls.bases is None:
return
for base in cls.bases:
if base in classes:
base_cls = classes[base]
self.include_methods_for_class(base_cls, classes)
for member in base_cls.members:
if isinstance(member, docspec.Function) and not any(m.name == member.name for m in cls.members):
new_member = copy.deepcopy(member)
new_member.parent = cls
cls.members.append(new_member)