mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-06-26 22:00:13 +00:00

* Initial commit, add search_engine * Add TopPSampler * Add more TopPSampler unit tests * Remove SearchEngineSampler (converted to TopPSampler) * Add some basic WebSearch unit tests * Rename unit tests * Add WebRetriever into agent_tools * Adjust to WebRetriever * Add WebRetriever mode [snippet|document] * Minor changes * SerperDev: add peopleAlsoAsk search results * First agent for hotpotqa * Making WebRetriever work on hotpotqa * refactor: minor WebRetriever improvements (#4377) * refactor: remove doc ids rebuild + antecipate cache * refactor: improve caching, fix Document ids * Minor WebRetriever improvements * Overlooked minor fixes * feat: add Bing API as search engine * refactor: let kwargs pass-through * feat: increase search context * check sampler result, improve batch typing * refactor: increase mypy compliance * Initial commit, add search_engine * Add TopPSampler * Add more TopPSampler unit tests * Remove SearchEngineSampler (converted to TopPSampler) * Add some basic WebSearch unit tests * Rename unit tests * Add WebRetriever into agent_tools * Adjust to WebRetriever * Add WebRetriever mode [snippet|document] * Minor changes * SerperDev: add peopleAlsoAsk search results * First agent for hotpotqa * Making WebRetriever work on hotpotqa * refactor: minor WebRetriever improvements (#4377) * refactor: remove doc ids rebuild + antecipate cache * refactor: improve caching, fix Document ids * Minor WebRetriever improvements * Overlooked minor fixes * feat: add Bing API as search engine * refactor: let kwargs pass-through * feat: increase search context * check sampler result, improve batch typing * refactor: increase mypy compliance * Fix mypy * Minor example fixes * Fix the descriptions * PR feedback updates * More fixes * TopPSampler: handle top p None value, add unit test * Add top_k to WebSearch * Use boilerpy3 instead trafilatura * Remove date finding * Add more WebRetriever docs * Refactor long methods * making the preprocessor optional * hide WebSearch and make NeuralWebSearch a pipeline * remove unused imports * add WebQAPipeline and split example into two * change example search engine to SerperDev * Turn off progress bars in WebRetriever's PreProcesssor * Agent tool examples - final updates * Add webqa test, search results ranking scores * Better answer box handling for SerperDev and SerpAPI * Minor fixes * pylint * pylint fixes * extract TopPSampler from WebRetriever * use sampler only for WebRetriever modes other than snippet * add web retriever tests * add web retriever tests * exclude rdflib@6.3.2 due to license issues * add test for preprocessed docs and kwargs examples in docstrings * Move test_webqa_pipeline to test/pipelines * change docstring for join_documents_and_scores * Use WebQAPipeline in examples/web_lfqa.py * Use WebQAPipeline in examples/web_lfqa.py * Move test_webqa_pipeline to e2e * Updated lg * Sampler added automatically in WebQAPipeline, no need to add it * Updated lg * Updated lg * :ignore Update agent tools examples to new templates (#4503) * Update examples to new templates * Add print back * fix linting and black format issues --------- Co-authored-by: Daniel Bichuetti <daniel.bichuetti@gmail.com> Co-authored-by: agnieszka-m <amarzec13@gmail.com> Co-authored-by: Julian Risch <julian.risch@deepset.ai>
36 lines
1.1 KiB
Python
36 lines
1.1 KiB
Python
import os
|
|
from haystack.nodes import PromptNode
|
|
from haystack.nodes.retriever.web import WebRetriever
|
|
from haystack.pipelines import WebQAPipeline
|
|
|
|
search_key = os.environ.get("SERPERDEV_API_KEY")
|
|
if not search_key:
|
|
raise ValueError("Please set the SERPERDEV_API_KEY environment variable")
|
|
|
|
openai_key = os.environ.get("OPENAI_API_KEY")
|
|
if not search_key:
|
|
raise ValueError("Please set the OPENAI_API_KEY environment variable")
|
|
|
|
prompt_node = PromptNode(
|
|
"text-davinci-003",
|
|
api_key=openai_key,
|
|
max_length=256,
|
|
default_prompt_template="question-answering-with-document-scores",
|
|
)
|
|
web_retriever = WebRetriever(api_key=search_key)
|
|
pipeline = WebQAPipeline(retriever=web_retriever, prompt_node=prompt_node)
|
|
|
|
questions = [
|
|
"Who won the 1971 San Francisco mayoral election?",
|
|
"Where was Jeremy McKinnon born?",
|
|
"What river is near Dundalk, Ireland?",
|
|
"Who is Kyle Moran?",
|
|
"What party does Joseph Alioto belong to?",
|
|
"When was the Democratic Party founded?",
|
|
"Who is Olivia Wilde's boyfriend?",
|
|
]
|
|
|
|
for question in questions:
|
|
response = pipeline.run(question)
|
|
print(f"{question} - {response['answers'][0].answer}")
|