docs: add Data Classes API ref and cleanup (#6482)

* Data Classes and cleanup

* Update pydoc-markdown.sh

* Update converter.yml

.github/utils/pydoc-markdown.sh

@@ -9,11 +9,3 @@ for file in ../config/* ; do
     echo "Converting $file..."
     pydoc-markdown "$file"
 done
-# render preview markdown docs
-cd ..
-rm -rf temp-preview && mkdir temp-preview
-cd temp-preview
-for file in ../config-preview/* ; do
-    echo "Converting $file..."
-    pydoc-markdown "$file"
-done

docs/pydoc/config/builder.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/builders]
+    search_path: [../../../haystack/components/builders]
     modules: ["answer_builder", "prompt_builder", "dynamic_prompt_builder"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/caching.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/caching]
+    search_path: [../../../haystack/components/caching]
     modules: ["url_cache_checker"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/classifier.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/classifiers]
+    search_path: [../../../haystack/components/classifiers]
     modules: ["document_language_classifier"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/converter.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/converters]
+    search_path: [../../../haystack/components/converters]
     modules: ["azure", "html", "markdown", "pypdf", "tika", "txt"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -13,11 +13,11 @@ processors:
   - type: crossref
 renderer:
   type: renderers.ReadmePreviewRenderer
-  excerpt: Extracts text from files in different formats and converts it into the unified Document format.
+  excerpt: Extracts text from files in different formats and converts it into a unified Document format.
   category_slug: haystack-classes
   title: Converter API
   slug: converter-api
-  order: 50
+  order: 20
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/dataclass.yml

@@ -0,0 +1,26 @@
+loaders:
+  - type: loaders.CustomPythonLoader
+    search_path: [../../../haystack/dataclasses]
+    modules: ["answer", "byte_steam", "chat_message", "document", "streaming_chunk"]
+    ignore_when_discovered: ["__init__"]
+processors:
+  - type: filter
+    expression:
+    documented_only: true
+    do_not_filter_modules: false
+    skip_empty_modules: true
+  - type: smart
+  - type: crossref
+renderer:
+  type: renderers.ReadmePreviewRenderer
+  excerpt: Core classes that carry data through the system.
+  category_slug: data-classes
+  title: Data Classes API
+  slug: data-classes-api
+  order: 30
+  markdown:
+    descriptive_class_title: false
+    descriptive_module_title: true
+    add_method_class_prefix: true
+    add_member_class_prefix: false
+    filename: data_classes_api.md

docs/pydoc/config/document_store.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/document_stores/in_memory]
+    search_path: [../../../haystack/document_stores/in_memory]
     modules: ["document_store"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: DocumentStore API
   slug: document-store-api
-  order: 20
+  order: 40
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/embedder.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/embedders]
+    search_path: [../../../haystack/components/embedders]
     modules: ["openai_document_embedder", "openai_text_embedder", "sentence_transformers_document_embedder", "sentence_transformers_text_embedder"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: Embedder API
   slug: embedder-api
-  order: 40
+  order: 60
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/fetcher.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/fetchers]
+    search_path: [../../../haystack/components/fetchers]
     modules: ["link_content"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: LinkContentFetcher API
   slug: fetcher-api
-  order: 70
+  order: 80
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/generator.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/generators]
+    search_path: [../../../haystack/components/generators]
     modules: ["hugging_face_local", "hugging_face_tgi", "openai", "chat/hugging_face_tgi", "chat/openai"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: Generator API
   slug: generator-api
-  order: 60
+  order: 70
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/pipeline.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview]
+    search_path: [../../../haystack]
     modules: ["pipeline"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: Pipelines API
   slug: pipelines-api
-  order: 80
+  order: 90
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/preprocessor.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/preprocessors]
+    search_path: [../../../haystack/components/preprocessors]
     modules: ["document_cleaner", "document_splitter"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: PreProcessor API
   slug: preprocessor-api
-  order: 90
+  order: 100
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true

docs/pydoc/config/ranker.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/rankers]
+    search_path: [../../../haystack/components/rankers]
     modules: ["meta_field", "transformers_similarity"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/reader.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/readers]
+    search_path: [../../../haystack/components/readers]
     modules: ["extractive"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/retriever.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/retrievers]
+    search_path: [../../../haystack/components/retrievers]
     modules: ["in_memory_bm25_retriever", "in_memory_embedding_retriever"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/router.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/routers]
+    search_path: [../../../haystack/components/routers]
     modules: ["document_joiner", "conditional_router", "file_type_router", "metadata_router", "text_language_router"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/sampler.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/samplers]
+    search_path: [../../../haystack/components/samplers]
     modules: ["top_p"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/websearch.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/websearch]
+    search_path: [../../../haystack/components/websearch]
     modules: ["serper_dev"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/whisper.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/audio]
+    search_path: [../../../haystack/components/audio]
     modules: ["whisper_local", "whisper_remote"]
     ignore_when_discovered: ["__init__"]
 processors:

docs/pydoc/config/writer.yml

@@ -1,6 +1,6 @@
 loaders:
   - type: loaders.CustomPythonLoader
-    search_path: [../../../haystack/preview/components/writers]
+    search_path: [../../../haystack/components/writers]
     modules: ["document_writer"]
     ignore_when_discovered: ["__init__"]
 processors:
@@ -17,7 +17,7 @@ renderer:
   category_slug: haystack-classes
   title: DocumentWriter API
   slug: writer-api
-  order: 30
+  order: 50
   markdown:
     descriptive_class_title: false
     descriptive_module_title: true