mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-10-17 02:48:30 +00:00
Fix building Pipeline with YAML (#800)
This commit is contained in:
parent
f3a3b73d9b
commit
7b18e324f2
@ -344,7 +344,7 @@ Return a summary of the documents in the document store
|
|||||||
#### update\_embeddings
|
#### update\_embeddings
|
||||||
|
|
||||||
```python
|
```python
|
||||||
| update_embeddings(retriever: BaseRetriever, index: Optional[str] = None, batch_size: int = 10_000)
|
| update_embeddings(retriever, index: Optional[str] = None, batch_size: int = 10_000)
|
||||||
```
|
```
|
||||||
|
|
||||||
Updates the embeddings in the the document store using the encoding model specified in the retriever.
|
Updates the embeddings in the the document store using the encoding model specified in the retriever.
|
||||||
|
@ -0,0 +1,5 @@
|
|||||||
|
from haystack.document_store.elasticsearch import ElasticsearchDocumentStore
|
||||||
|
from haystack.document_store.faiss import FAISSDocumentStore
|
||||||
|
from haystack.document_store.memory import InMemoryDocumentStore
|
||||||
|
from haystack.document_store.milvus import MilvusDocumentStore
|
||||||
|
from haystack.document_store.sql import SQLDocumentStore
|
@ -12,7 +12,6 @@ from scipy.special import expit
|
|||||||
|
|
||||||
from haystack.document_store.base import BaseDocumentStore
|
from haystack.document_store.base import BaseDocumentStore
|
||||||
from haystack import Document, Label
|
from haystack import Document, Label
|
||||||
from haystack.retriever.base import BaseRetriever
|
|
||||||
from haystack.utils import get_batches_from_generator
|
from haystack.utils import get_batches_from_generator
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@ -755,7 +754,7 @@ class ElasticsearchDocumentStore(BaseDocumentStore):
|
|||||||
}
|
}
|
||||||
return stats
|
return stats
|
||||||
|
|
||||||
def update_embeddings(self, retriever: BaseRetriever, index: Optional[str] = None, batch_size: int = 10_000):
|
def update_embeddings(self, retriever, index: Optional[str] = None, batch_size: int = 10_000):
|
||||||
"""
|
"""
|
||||||
Updates the embeddings in the the document store using the encoding model specified in the retriever.
|
Updates the embeddings in the the document store using the encoding model specified in the retriever.
|
||||||
This can be useful if want to add or change the embeddings for your documents (e.g. after changing the retriever config).
|
This can be useful if want to add or change the embeddings for your documents (e.g. after changing the retriever config).
|
||||||
|
@ -0,0 +1,4 @@
|
|||||||
|
from haystack.file_converter.docx import DocxToTextConverter
|
||||||
|
from haystack.file_converter.pdf import PDFToTextConverter
|
||||||
|
from haystack.file_converter.tika import TikaConverter
|
||||||
|
from haystack.file_converter.txt import TextConverter
|
@ -0,0 +1 @@
|
|||||||
|
from haystack.generator.transformers import RAGenerator
|
@ -1,7 +1,7 @@
|
|||||||
import os
|
import os
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List, Optional, Dict, Type
|
from typing import List, Optional, Dict
|
||||||
|
|
||||||
import networkx as nx
|
import networkx as nx
|
||||||
import yaml
|
import yaml
|
||||||
@ -9,7 +9,6 @@ from networkx import DiGraph
|
|||||||
from networkx.drawing.nx_agraph import to_agraph
|
from networkx.drawing.nx_agraph import to_agraph
|
||||||
|
|
||||||
from haystack import BaseComponent
|
from haystack import BaseComponent
|
||||||
from haystack.document_store.base import BaseDocumentStore
|
|
||||||
from haystack.generator.base import BaseGenerator
|
from haystack.generator.base import BaseGenerator
|
||||||
from haystack.reader.base import BaseReader
|
from haystack.reader.base import BaseReader
|
||||||
from haystack.retriever.base import BaseRetriever
|
from haystack.retriever.base import BaseRetriever
|
||||||
@ -240,19 +239,7 @@ class Pipeline:
|
|||||||
cls._load_or_get_component(name=value, definitions=definitions, components=components)
|
cls._load_or_get_component(name=value, definitions=definitions, components=components)
|
||||||
component_params[key] = components[value] # substitute reference (string) with the component object.
|
component_params[key] = components[value] # substitute reference (string) with the component object.
|
||||||
|
|
||||||
if "DocumentStore" in component_type:
|
instance = BaseComponent.load_from_args(component_type=component_type, **component_params)
|
||||||
ComponentClass: Type[BaseComponent] = BaseDocumentStore
|
|
||||||
elif "Reader" in component_type:
|
|
||||||
ComponentClass = BaseReader
|
|
||||||
elif "Retriever" in component_type:
|
|
||||||
ComponentClass = BaseRetriever
|
|
||||||
elif "Generator" in component_type:
|
|
||||||
ComponentClass = BaseGenerator
|
|
||||||
elif "Summarizer" in component_type:
|
|
||||||
ComponentClass = BaseSummarizer
|
|
||||||
else:
|
|
||||||
raise NotImplementedError(f"Component of type '{component_type}' is not implemented for pipelines.")
|
|
||||||
instance = ComponentClass.load_from_args(component_type=component_type, **component_params)
|
|
||||||
components[name] = instance
|
components[name] = instance
|
||||||
return instance
|
return instance
|
||||||
|
|
||||||
|
@ -0,0 +1 @@
|
|||||||
|
from haystack.preprocessor.preprocessor import PreProcessor
|
@ -0,0 +1,2 @@
|
|||||||
|
from haystack.reader.farm import FARMReader
|
||||||
|
from haystack.reader.transformers import TransformersReader
|
@ -0,0 +1,2 @@
|
|||||||
|
from haystack.retriever.dense import DensePassageRetriever, EmbeddingRetriever
|
||||||
|
from haystack.retriever.sparse import ElasticsearchRetriever
|
@ -233,5 +233,7 @@ class BaseComponent:
|
|||||||
:param component_type: name of the component class to load.
|
:param component_type: name of the component class to load.
|
||||||
:param kwargs: parameters to pass to the __init__() for the component.
|
:param kwargs: parameters to pass to the __init__() for the component.
|
||||||
"""
|
"""
|
||||||
|
if component_type not in cls.subclasses.keys():
|
||||||
|
raise Exception(f"Haystack component with the name '{component_type}' does not exist.")
|
||||||
instance = cls.subclasses[component_type](**kwargs)
|
instance = cls.subclasses[component_type](**kwargs)
|
||||||
return instance
|
return instance
|
||||||
|
@ -0,0 +1 @@
|
|||||||
|
from haystack.summarizer.transformers import TransformersSummarizer
|
Loading…
x
Reference in New Issue
Block a user