mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-25 09:50:14 +00:00
ac5617e757
* add basic telemetry features * change pipeline_config to _component_config * Update Documentation & Code Style * add super().__init__() calls to error classes * make posthog mock work with python 3.7 * Update Documentation & Code Style * update link to docs web page * log exceptions, send event for raised HaystackErrors, refactor Path(CONFIG_PATH) * add comment on send_event in BaseComponent.init() and fix mypy * mock NonPrivateParameters and fix pylint undefined-variable * Update Documentation & Code Style * check model path contains multiple / * add test for writing to file * add test for en-/disable telemetry * Update Documentation & Code Style * merge file deletion methods and ignore pylint global statement * Update Documentation & Code Style * set env variable in demo to activate telemetry * fix mock of HAYSTACK_TELEMETRY_ENABLED * fix mypy and linter * add CI as env variable to execution contexts * remove threading, add test for custom error event * Update Documentation & Code Style * simplify config/log file deletion * add test for final event being sent * force writing config file in test * make test compatible with python 3.7 * switch to posthog production server * Update Documentation & Code Style Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
103 lines
4.3 KiB
Python
103 lines
4.3 KiB
Python
from haystack.utils import convert_files_to_dicts, fetch_archive_from_http, clean_wiki_text
|
|
from haystack.nodes import Seq2SeqGenerator
|
|
|
|
|
|
def tutorial12_lfqa():
|
|
|
|
"""
|
|
Document Store:
|
|
FAISS is a library for efficient similarity search on a cluster of dense vectors.
|
|
The `FAISSDocumentStore` uses a SQL(SQLite in-memory be default) database under-the-hood
|
|
to store the document text and other meta data. The vector embeddings of the text are
|
|
indexed on a FAISS Index that later is queried for searching answers.
|
|
The default flavour of FAISSDocumentStore is "Flat" but can also be set to "HNSW" for
|
|
faster search at the expense of some accuracy. Just set the faiss_index_factor_str argument in the constructor.
|
|
For more info on which suits your use case: https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index
|
|
"""
|
|
|
|
from haystack.document_stores.faiss import FAISSDocumentStore
|
|
|
|
document_store = FAISSDocumentStore(embedding_dim=128, faiss_index_factory_str="Flat")
|
|
|
|
"""
|
|
Cleaning & indexing documents:
|
|
Similarly to the previous tutorials, we download, convert and index some Game of Thrones articles to our DocumentStore
|
|
"""
|
|
|
|
# Let's first get some files that we want to use
|
|
doc_dir = "data/tutorial12"
|
|
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt12.zip"
|
|
fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
|
|
|
|
# Convert files to dicts
|
|
dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True)
|
|
|
|
# Now, let's write the dicts containing documents to our DB.
|
|
document_store.write_documents(dicts)
|
|
|
|
"""
|
|
Initalize Retriever and Reader/Generator:
|
|
We use a `DensePassageRetriever` and we invoke `update_embeddings` to index the embeddings of documents in the `FAISSDocumentStore`
|
|
"""
|
|
|
|
from haystack.nodes import DensePassageRetriever
|
|
|
|
retriever = DensePassageRetriever(
|
|
document_store=document_store,
|
|
query_embedding_model="vblagoje/dpr-question_encoder-single-lfqa-wiki",
|
|
passage_embedding_model="vblagoje/dpr-ctx_encoder-single-lfqa-wiki",
|
|
)
|
|
|
|
document_store.update_embeddings(retriever)
|
|
|
|
"""Before we blindly use the `DensePassageRetriever` let's empirically test it to make sure a simple search indeed finds the relevant documents."""
|
|
|
|
from haystack.utils import print_documents
|
|
from haystack.pipelines import DocumentSearchPipeline
|
|
|
|
p_retrieval = DocumentSearchPipeline(retriever)
|
|
res = p_retrieval.run(query="Tell me something about Arya Stark?", params={"Retriever": {"top_k": 1}})
|
|
print_documents(res, max_text_len=512)
|
|
|
|
"""
|
|
Similar to previous Tutorials we now initalize our reader/generator.
|
|
Here we use a `Seq2SeqGenerator` with the *vblagoje/bart_lfqa* model (see: https://huggingface.co/vblagoje/bart_lfqa)
|
|
"""
|
|
|
|
generator = Seq2SeqGenerator(model_name_or_path="vblagoje/bart_lfqa")
|
|
|
|
"""
|
|
Pipeline:
|
|
With a Haystack `Pipeline` you can stick together your building blocks to a search pipeline.
|
|
Under the hood, `Pipelines` are Directed Acyclic Graphs (DAGs) that you can easily customize for your own use cases.
|
|
To speed things up, Haystack also comes with a few predefined Pipelines. One of them is the `GenerativeQAPipeline` that combines a retriever and a reader/generator to answer our questions.
|
|
You can learn more about `Pipelines` in the [docs](https://haystack.deepset.ai/docs/latest/pipelinesmd).
|
|
"""
|
|
|
|
from haystack.pipelines import GenerativeQAPipeline
|
|
|
|
pipe = GenerativeQAPipeline(generator, retriever)
|
|
|
|
"""Voilà! Ask a question!"""
|
|
|
|
query_1 = "How did Arya Stark's character get portrayed in a television adaptation?"
|
|
result_1 = pipe.run(query=query_1, params={"Retriever": {"top_k": 3}})
|
|
print(f"Query: {query_1}")
|
|
print(f"Answer: {result_1['answers'][0]}")
|
|
print()
|
|
|
|
query_2 = "Why is Arya Stark an unusual character?"
|
|
result_2 = pipe.run(query=query_2, params={"Retriever": {"top_k": 3}})
|
|
print(f"Query: {query_2}")
|
|
print(f"Answer: {result_2['answers'][0]}")
|
|
print()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
tutorial12_lfqa()
|
|
|
|
|
|
# This Haystack script was made with love by deepset in Berlin, Germany
|
|
# Haystack: https://github.com/deepset-ai/haystack
|
|
# deepset: https://deepset.ai/
|