mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-23 08:52:16 +00:00

* Files moved, imports all broken * Fix most imports and docstrings into * Fix the paths to the modules in the API docs * Add latest docstring and tutorial changes * Add a few pipelines that were lost in the inports * Fix a bunch of mypy warnings * Add latest docstring and tutorial changes * Create a file_classifier module * Add docs for file_classifier * Fixed most circular imports, now the REST API can start * Add latest docstring and tutorial changes * Tackling more mypy issues * Reintroduce from FARM and fix last mypy issues hopefully * Re-enable old-style imports * Fix some more import from the top-level package in an attempt to sort out circular imports * Fix some imports in tests to new-style to prevent failed class equalities from breaking tests * Change document_store into document_stores * Update imports in tutorials * Add latest docstring and tutorial changes * Probably fixes summarizer tests * Improve the old-style import allowing module imports (should work) * Try to fix the docs * Remove dedicated KnowledgeGraph page from autodocs * Remove dedicated GraphRetriever page from autodocs * Fix generate_docstrings.sh with an updated list of yaml files to look for * Fix some more modules in the docs * Fix the document stores docs too * Fix a small issue on Tutorial14 * Add latest docstring and tutorial changes * Add deprecation warning to old-style imports * Remove stray folder and import Dict into dense.py * Change import path for MLFlowLogger * Add old loggers path to the import path aliases * Fix debug output of convert_ipynb.py * Fix circular import on BaseRetriever * Missed one merge block * re-run tutorial 5 * Fix imports in tutorial 5 * Re-enable squad_to_dpr CLI from the root package and move get_batches_from_generator into document_stores.base * Add latest docstring and tutorial changes * Fix typo in utils __init__ * Fix a few more imports * Fix benchmarks too * New-style imports in test_knowledge_graph * Rollback setup.py * Rollback squad_to_dpr too Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
78 lines
4.2 KiB
Python
Executable File
78 lines
4.2 KiB
Python
Executable File
from haystack.document_stores import FAISSDocumentStore, MilvusDocumentStore
|
|
from haystack.utils import clean_wiki_text, print_answers, launch_milvus, convert_files_to_dicts, fetch_archive_from_http
|
|
from haystack.nodes import FARMReader, DensePassageRetriever
|
|
|
|
def tutorial6_better_retrieval_via_dpr():
|
|
# OPTION 1: FAISS is a library for efficient similarity search on a cluster of dense vectors.
|
|
# The FAISSDocumentStore uses a SQL(SQLite in-memory be default) document store under-the-hood
|
|
# to store the document text and other meta data. The vector embeddings of the text are
|
|
# indexed on a FAISS Index that later is queried for searching answers.
|
|
# The default flavour of FAISSDocumentStore is "Flat" but can also be set to "HNSW" for
|
|
# faster search at the expense of some accuracy. Just set the faiss_index_factor_str argument in the constructor.
|
|
# For more info on which suits your use case: https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index
|
|
document_store = FAISSDocumentStore(faiss_index_factory_str="Flat")
|
|
|
|
# OPTION2: Milvus is an open source database library that is also optimized for vector similarity searches like FAISS.
|
|
# Like FAISS it has both a "Flat" and "HNSW" mode but it outperforms FAISS when it comes to dynamic data management.
|
|
# It does require a little more setup, however, as it is run through Docker and requires the setup of some config files.
|
|
# See https://milvus.io/docs/v1.0.0/milvus_docker-cpu.md
|
|
# launch_milvus()
|
|
# document_store = MilvusDocumentStore()
|
|
|
|
# ## Preprocessing of documents
|
|
# Let's first get some documents that we want to query
|
|
doc_dir = "data/article_txt_got"
|
|
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip"
|
|
fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
|
|
|
|
# convert files to dicts containing documents that can be indexed to our datastore
|
|
dicts = convert_files_to_dicts(dir_path=doc_dir, clean_func=clean_wiki_text, split_paragraphs=True)
|
|
|
|
# Now, let's write the docs to our DB.
|
|
document_store.write_documents(dicts)
|
|
|
|
### Retriever
|
|
retriever = DensePassageRetriever(document_store=document_store,
|
|
query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
|
|
passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
|
|
max_seq_len_query=64,
|
|
max_seq_len_passage=256,
|
|
batch_size=2,
|
|
use_gpu=True,
|
|
embed_title=True,
|
|
use_fast_tokenizers=True
|
|
)
|
|
|
|
# Important:
|
|
# Now that after we have the DPR initialized, we need to call update_embeddings() to iterate over all
|
|
# previously indexed documents and update their embedding representation.
|
|
# While this can be a time consuming operation (depending on corpus size), it only needs to be done once.
|
|
# At query time, we only need to embed the query and compare it the existing doc embeddings which is very fast.
|
|
document_store.update_embeddings(retriever)
|
|
|
|
### Reader
|
|
# Load a local model or any of the QA models on
|
|
# Hugging Face's model hub (https://huggingface.co/models)
|
|
reader = FARMReader(model_name_or_path="deepset/roberta-base-squad2", use_gpu=True)
|
|
|
|
### Pipeline
|
|
from haystack.pipelines import ExtractiveQAPipeline
|
|
pipe = ExtractiveQAPipeline(reader, retriever)
|
|
|
|
## Voilà! Ask a question!
|
|
prediction = pipe.run(
|
|
query="Who is the father of Arya Stark?", params={"Retriever": {"top_k": 10}, "Reader": {"top_k": 5}}
|
|
)
|
|
|
|
# prediction = pipe.run(query="Who created the Dothraki vocabulary?", params={"Reader": {"top_k": 5}})
|
|
# prediction = pipe.run(query="Who is the sister of Sansa?", params={"Reader": {"top_k": 5}})
|
|
|
|
print_answers(prediction, details="minimal")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
tutorial6_better_retrieval_via_dpr()
|
|
|
|
# This Haystack script was made with love by deepset in Berlin, Germany
|
|
# Haystack: https://github.com/deepset-ai/haystack
|
|
# deepset: https://deepset.ai/ |