From a26859f065aa53c5cd7e45e33f7067b52b63be8a Mon Sep 17 00:00:00 2001 From: bogdankostic Date: Wed, 2 Aug 2023 18:54:15 +0200 Subject: [PATCH] docs: Add inherited methods to API reference documentation (#5273) * Add inherited methods to API reference documentation * Fix typing --- docs/pydoc/config/agent.yml | 2 +- docs/pydoc/config/answer-generator.yml | 2 +- docs/pydoc/config/crawler.yml | 3 +- docs/pydoc/config/doc-language-classifier.yml | 2 +- docs/pydoc/config/document-classifier.yml | 3 +- docs/pydoc/config/document-store.yml | 2 +- docs/pydoc/config/extractor.yml | 3 +- docs/pydoc/config/file-classifier.yml | 3 +- docs/pydoc/config/file-converters.yml | 2 +- docs/pydoc/config/other.yml | 3 +- docs/pydoc/config/pipelines.yml | 2 +- docs/pydoc/config/preprocessor.yml | 3 +- docs/pydoc/config/primitives.yml | 3 +- docs/pydoc/config/prompt-node.yml | 2 +- docs/pydoc/config/pseudo-label-generator.yml | 3 +- docs/pydoc/config/query-classifier.yml | 3 +- docs/pydoc/config/question-generator.yml | 3 +- docs/pydoc/config/ranker.yml | 2 +- docs/pydoc/config/reader.yml | 3 +- docs/pydoc/config/retriever.yml | 2 +- docs/pydoc/config/shaper.yml | 2 +- docs/pydoc/config/summarizer.yml | 2 +- .../pydoc/config/transformers-img-to-text.yml | 3 +- docs/pydoc/config/translator.yml | 3 +- docs/pydoc/config/utils.yml | 2 +- docs/pydoc/config/whisper.yml | 2 +- docs/pydoc/loaders.py | 63 +++++++++++++++++++ 27 files changed, 89 insertions(+), 39 deletions(-) create mode 100644 docs/pydoc/loaders.py diff --git a/docs/pydoc/config/agent.yml b/docs/pydoc/config/agent.yml index 80cd833e7..20dcf2be7 100644 --- a/docs/pydoc/config/agent.yml +++ b/docs/pydoc/config/agent.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/agents] modules: ['base', 'conversational', 'utils', 'agent_step', 'memory/conversation_memory', 'memory/conversation_summary_memory'] ignore_when_discovered: ['__init__'] diff --git a/docs/pydoc/config/answer-generator.yml b/docs/pydoc/config/answer-generator.yml index 817a5709a..695ad38e4 100644 --- a/docs/pydoc/config/answer-generator.yml +++ b/docs/pydoc/config/answer-generator.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/answer_generator] modules: ["base", "openai"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/config/crawler.yml b/docs/pydoc/config/crawler.yml index 6528da857..657e2f989 100644 --- a/docs/pydoc/config/crawler.yml +++ b/docs/pydoc/config/crawler.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/connector] modules: ["crawler"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: crawler_api.md - diff --git a/docs/pydoc/config/doc-language-classifier.yml b/docs/pydoc/config/doc-language-classifier.yml index 040afe8fa..acdc7f939 100644 --- a/docs/pydoc/config/doc-language-classifier.yml +++ b/docs/pydoc/config/doc-language-classifier.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/doc_language_classifier] modules: ["base", "langdetect", "transformers"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/config/document-classifier.yml b/docs/pydoc/config/document-classifier.yml index 0ade0897d..97be9d655 100644 --- a/docs/pydoc/config/document-classifier.yml +++ b/docs/pydoc/config/document-classifier.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/document_classifier] modules: ["base", "transformers"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: document_classifier_api.md - diff --git a/docs/pydoc/config/document-store.yml b/docs/pydoc/config/document-store.yml index 4e9a4a1b0..31cd08a4c 100644 --- a/docs/pydoc/config/document-store.yml +++ b/docs/pydoc/config/document-store.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/document_stores, ../../../haystack/document_stores/elasticsearch] modules: [ diff --git a/docs/pydoc/config/extractor.yml b/docs/pydoc/config/extractor.yml index bfa121d4b..13249dd8c 100644 --- a/docs/pydoc/config/extractor.yml +++ b/docs/pydoc/config/extractor.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/extractor] modules: ["entity"] ignore_when_discovered: ["__init__"] @@ -26,4 +26,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: entity_extractor_api.md - diff --git a/docs/pydoc/config/file-classifier.yml b/docs/pydoc/config/file-classifier.yml index 2b8872629..d916b9c22 100644 --- a/docs/pydoc/config/file-classifier.yml +++ b/docs/pydoc/config/file-classifier.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/file_classifier] modules: ["file_type"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: file_classifier_api.md - diff --git a/docs/pydoc/config/file-converters.yml b/docs/pydoc/config/file-converters.yml index 4a4182bd0..6dbe1ec74 100644 --- a/docs/pydoc/config/file-converters.yml +++ b/docs/pydoc/config/file-converters.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/file_converter] modules: [ diff --git a/docs/pydoc/config/other.yml b/docs/pydoc/config/other.yml index 0aa453376..a9f2f04d6 100644 --- a/docs/pydoc/config/other.yml +++ b/docs/pydoc/config/other.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/other] modules: [ @@ -32,4 +32,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: other_api.md - diff --git a/docs/pydoc/config/pipelines.yml b/docs/pydoc/config/pipelines.yml index 1b49cd03b..609d32913 100644 --- a/docs/pydoc/config/pipelines.yml +++ b/docs/pydoc/config/pipelines.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/pipelines] modules: ["base", "ray", "standard_pipelines"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/config/preprocessor.yml b/docs/pydoc/config/preprocessor.yml index 1eb437aea..f6a86113e 100644 --- a/docs/pydoc/config/preprocessor.yml +++ b/docs/pydoc/config/preprocessor.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/preprocessor] modules: ["base", "preprocessor"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: preprocessor_api.md - diff --git a/docs/pydoc/config/primitives.yml b/docs/pydoc/config/primitives.yml index 6b8c3c8d1..c689eec60 100644 --- a/docs/pydoc/config/primitives.yml +++ b/docs/pydoc/config/primitives.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack] modules: ["schema"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: primitives_api.md - diff --git a/docs/pydoc/config/prompt-node.yml b/docs/pydoc/config/prompt-node.yml index bd67c933f..e2b5e3704 100644 --- a/docs/pydoc/config/prompt-node.yml +++ b/docs/pydoc/config/prompt-node.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/prompt] modules: [ diff --git a/docs/pydoc/config/pseudo-label-generator.yml b/docs/pydoc/config/pseudo-label-generator.yml index e1df263fd..28f645dee 100644 --- a/docs/pydoc/config/pseudo-label-generator.yml +++ b/docs/pydoc/config/pseudo-label-generator.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/label_generator] modules: ["pseudo_label_generator"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: pseudo_label_generator_api.md - diff --git a/docs/pydoc/config/query-classifier.yml b/docs/pydoc/config/query-classifier.yml index 04c8b19a2..8e7b855a5 100644 --- a/docs/pydoc/config/query-classifier.yml +++ b/docs/pydoc/config/query-classifier.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/query_classifier] modules: ["base", "sklearn", "transformers"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: query_classifier_api.md - diff --git a/docs/pydoc/config/question-generator.yml b/docs/pydoc/config/question-generator.yml index 72a0c5c22..4a3f9a64f 100644 --- a/docs/pydoc/config/question-generator.yml +++ b/docs/pydoc/config/question-generator.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/question_generator] modules: ["question_generator"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: question_generator_api.md - diff --git a/docs/pydoc/config/ranker.yml b/docs/pydoc/config/ranker.yml index 052756615..206a6f0d7 100644 --- a/docs/pydoc/config/ranker.yml +++ b/docs/pydoc/config/ranker.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/ranker] modules: ["base", "sentence_transformers", "recentness_ranker", "diversity", "lost_in_the_middle"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/config/reader.yml b/docs/pydoc/config/reader.yml index a36efd109..013f9cae6 100644 --- a/docs/pydoc/config/reader.yml +++ b/docs/pydoc/config/reader.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/reader] modules: ["base", "farm", "transformers", "table"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: reader_api.md - diff --git a/docs/pydoc/config/retriever.yml b/docs/pydoc/config/retriever.yml index 4b0f701c8..3003f7892 100644 --- a/docs/pydoc/config/retriever.yml +++ b/docs/pydoc/config/retriever.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/retriever] modules: ["base", "sparse", "dense", "multimodal/retriever", "web", "link_content"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/config/shaper.yml b/docs/pydoc/config/shaper.yml index 3970ce922..b2d5c8b80 100644 --- a/docs/pydoc/config/shaper.yml +++ b/docs/pydoc/config/shaper.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/other] modules: ['shaper'] ignore_when_discovered: ['__init__'] diff --git a/docs/pydoc/config/summarizer.yml b/docs/pydoc/config/summarizer.yml index 4fcc66655..c0045c9c6 100644 --- a/docs/pydoc/config/summarizer.yml +++ b/docs/pydoc/config/summarizer.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/summarizer] modules: ["base", "transformers"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/config/transformers-img-to-text.yml b/docs/pydoc/config/transformers-img-to-text.yml index 36ad30856..f58b0a777 100644 --- a/docs/pydoc/config/transformers-img-to-text.yml +++ b/docs/pydoc/config/transformers-img-to-text.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/image_to_text] modules: ["base", "transformers"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: transformers-image-to-text.md - diff --git a/docs/pydoc/config/translator.yml b/docs/pydoc/config/translator.yml index ee8a90318..41e8af4d4 100644 --- a/docs/pydoc/config/translator.yml +++ b/docs/pydoc/config/translator.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/translator] modules: ["base", "transformers"] ignore_when_discovered: ["__init__"] @@ -24,4 +24,3 @@ renderer: add_method_class_prefix: true add_member_class_prefix: false filename: translator_api.md - diff --git a/docs/pydoc/config/utils.yml b/docs/pydoc/config/utils.yml index bd7b4dd49..ddd044fd9 100644 --- a/docs/pydoc/config/utils.yml +++ b/docs/pydoc/config/utils.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/utils] modules: [ diff --git a/docs/pydoc/config/whisper.yml b/docs/pydoc/config/whisper.yml index 0c7fbab52..5b8b4324d 100644 --- a/docs/pydoc/config/whisper.yml +++ b/docs/pydoc/config/whisper.yml @@ -1,5 +1,5 @@ loaders: - - type: python + - type: loaders.CustomPythonLoader search_path: [../../../haystack/nodes/audio] modules: ["whisper_transcriber"] ignore_when_discovered: ["__init__"] diff --git a/docs/pydoc/loaders.py b/docs/pydoc/loaders.py new file mode 100644 index 000000000..0df8344cd --- /dev/null +++ b/docs/pydoc/loaders.py @@ -0,0 +1,63 @@ +import copy +import typing as t + +import docspec +from pydoc_markdown.interfaces import Context +from pydoc_markdown.contrib.loaders.python import PythonLoader + + +class CustomPythonLoader(PythonLoader): + def load(self) -> t.Iterable[docspec.Module]: + """ + Load the modules, but include inherited methods in the classes. + """ + # Load all haystack modules + temp_loader = PythonLoader(search_path=["../../../haystack"]) + temp_loader.init(Context(directory=".")) + all_modules = list(temp_loader.load()) + + # Collect all classes + classes = {} + for module in all_modules: + for member in module.members: + if isinstance(member, docspec.Class): + classes[member.name] = member + + # Load the modules specified in the search path + modules = super().load() + + # Add inherited methods to the classes + modules = self.include_inherited_methods(modules, classes) + + return modules + + def include_inherited_methods( + self, modules: t.Iterable[docspec.Module], classes: t.Dict[str, docspec.Class] + ) -> t.Iterable[docspec.Module]: + """ + Recursively include inherited methods from the base classes. + """ + modules = list(modules) + for module in modules: + for cls in module.members: + if isinstance(cls, docspec.Class): + self.include_methods_for_class(cls, classes) + + return modules + + def include_methods_for_class(self, cls: docspec.Class, classes: t.Dict[str, docspec.Class]): + """ + Include all methods inherited from base classes to the class. + """ + if cls.bases is None: + return + for base in cls.bases: + if base in classes: + base_cls = classes[base] + self.include_methods_for_class(base_cls, classes) + + for member in base_cls.members: + if isinstance(member, docspec.Function) and not any(m.name == member.name for m in cls.members): + new_member = copy.deepcopy(member) + new_member.parent = cls + cls.members.append(new_member)