mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-07-16 05:20:51 +00:00
294 lines
11 KiB
Plaintext
294 lines
11 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Task: Question Answering for Game of Thrones\n",
|
|
"\n",
|
|
"<img style=\"float: right;\" src=\"https://upload.wikimedia.org/wikipedia/en/d/d8/Game_of_Thrones_title_card.jpg\">\n",
|
|
"\n",
|
|
"Question Answering can be used in a variety of use cases. A very common one: Using it to navigate through complex knowledge bases or long documents (\"search setting\").\n",
|
|
"\n",
|
|
"A \"knowledge base\" could for example be your website, an internal wiki or a collection of financial reports. \n",
|
|
"In this tutorial we will work on a slightly different domain: \"Game of Thrones\". \n",
|
|
"\n",
|
|
"Let's see how we can use a bunch of wikipedia articles to answer a variety of questions about the \n",
|
|
"marvellous seven kingdoms... \n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"# Let's start by adjust the working directory so that it is the root of the repository\n",
|
|
"# This should be run just once.\n",
|
|
"import os\n",
|
|
"os.chdir('../')\n",
|
|
"print(\"Current working directory is {}\".format(os.getcwd()))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from haystack import Finder\n",
|
|
"from haystack.database.sql import SQLDocumentStore\n",
|
|
"from haystack.indexing.cleaning import clean_wiki_text\n",
|
|
"from haystack.indexing.io import write_documents_to_db, fetch_archive_from_http\n",
|
|
"from haystack.reader.farm import FARMReader\n",
|
|
"from haystack.reader.transformers import TransformersReader\n",
|
|
"from haystack.retriever.tfidf import TfidfRetriever\n",
|
|
"from haystack.utils import print_answers"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Indexing & cleaning documents"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"11/28/2019 12:02:51 - INFO - haystack.indexing.io - Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip to `data/article_txt_got`\n",
|
|
"100%|██████████| 1167348/1167348 [00:00<00:00, 8157729.91B/s]\n",
|
|
"11/28/2019 12:02:52 - INFO - haystack.indexing.io - Wrote 517 docs to DB\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# Let's first get some documents that we want to query\n",
|
|
"# Here: 517 Wikipedia articles for Game of Thrones\n",
|
|
"doc_dir = \"data/article_txt_got\"\n",
|
|
"s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip\"\n",
|
|
"fetch_archive_from_http(url=s3_url, output_dir=doc_dir)\n",
|
|
"\n",
|
|
"# The documents can be stored in different types of \"DocumentStores\".\n",
|
|
"# For dev we suggest a light-weight SQL DB\n",
|
|
"# For production we suggest elasticsearch\n",
|
|
"document_store = SQLDocumentStore(url=\"sqlite:///qa.db\")\n",
|
|
"\n",
|
|
"# Now, let's write the docs to our DB.\n",
|
|
"# You can optionally supply a cleaning function that is applied to each doc (e.g. to remove footers)\n",
|
|
"# It must take a str as input, and return a str.\n",
|
|
"write_documents_to_db(document_store=document_store, document_dir=doc_dir, clean_func=clean_wiki_text, only_empty_db=True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Initalize Reader, Retriever & Finder"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 4,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false,
|
|
"name": "#%%\n"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"11/28/2019 12:02:56 - INFO - haystack.retriever.tfidf - Found 2813 candidate paragraphs from 519 docs in DB\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# A retriever identifies the k most promising chunks of text that might contain the answer for our question\n",
|
|
"# Retrievers use some simple but fast algorithm, here: TF-IDF\n",
|
|
"retriever = TfidfRetriever(document_store=document_store)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 5,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"11/28/2019 12:02:57 - INFO - haystack.indexing.io - Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-models/0.3.0/bert-english-qa-large.tar.gz to `model`\n",
|
|
"100%|██████████| 1245547135/1245547135 [00:53<00:00, 23191957.12B/s]\n",
|
|
"11/28/2019 12:04:05 - WARNING - farm.data_handler.processor - Loading tokenizer from deprecated FARM config. If you used `custom_vocab` or `never_split_chars`, this won't work anymore.\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# A reader scans the text chunks in detail and extracts the k best answers\n",
|
|
"# Reader use more powerful but slower deep learning models\n",
|
|
"# You can select a local model or any of the QA models published on huggingface's model hub (https://huggingface.co/models)\n",
|
|
"# here: a medium sized BERT QA model trained via FARM on Squad 2.0\n",
|
|
"reader = FARMReader(model_name_or_path=\"deepset/bert-base-cased-squad2\", use_gpu=False)\n",
|
|
"\n",
|
|
"# OR: use alternatively a reader from huggingface's transformers package (https://github.com/huggingface/transformers)\n",
|
|
"# reader = TransformersReader(model=\"distilbert-base-uncased-distilled-squad\", tokenizer=\"distilbert-base-uncased\", use_gpu=-1)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false
|
|
}
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"# The Finder sticks together retriever and retriever in a pipeline to answer our actual questions\n",
|
|
"finder = Finder(reader, retriever)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Voilá! Ask a question!"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 7,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stderr",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"11/28/2019 12:04:26 - INFO - haystack - Identified 10 candidates via retriever:\n",
|
|
" paragraph_id document_id text\n",
|
|
" 2723 506 \\n===Arya Stark===\\n'''Arya Stark''' portrayed by Maisie Williams. Arya Star...\n",
|
|
" 2212 407 \\n====Season 8====\\nArya reunites with Jon, Gendry, and the Hound, who have ...\n",
|
|
" 2205 407 \\n====Season 1====\\nArya accompanies her father Ned and her sister Sansa to ...\n",
|
|
" 548 105 \\n===''A Game of Thrones''===\\nSansa Stark begins the novel by being betroth...\n",
|
|
" 1437 258 \\n===In Braavos===\\nLady Crane returns to her chambers to find a wounded Ary...\n",
|
|
" 462 92 \\n== Characters ==\\nThe tale is told through the eyes of 9 recurring POV cha...\n",
|
|
" 2198 407 \\n==== ''A Game of Thrones'' ====\\nArya adopts a direwolf cub, which she nam...\n",
|
|
" 2211 407 \\n====Season 7====\\nTaking the face of Walder Frey, Arya gathers the men of ...\n",
|
|
" 570 106 \\n=== Arya Stark ===\\nArya Stark is the third child and younger daughter of ...\n",
|
|
" 313 65 \\n===On the Kingsroad===\\nCity Watchmen search the caravan for Gendry but ar...\n",
|
|
"11/28/2019 12:04:27 - INFO - haystack - Applying the reader now to look for the answer in detail ...\n",
|
|
"Inferencing: 100%|██████████| 1/1 [00:21<00:00, 21.82s/it]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"# You can configure how many candidates the reader and retriever shall return\n",
|
|
"# The higher top_k_retriever, the better (but also the slower) your answers. \n",
|
|
"prediction = finder.get_answers(question=\"Who is the father of Arya Stark?\", top_k_retriever=10, top_k_reader=5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"#prediction = finder.get_answers(question=\"Who created the Dothraki vocabulary?\", top_k_reader=5)\n",
|
|
"#prediction = finder.get_answers(question=\"Who is the sister of Sansa?\", top_k_reader=5)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {
|
|
"pycharm": {
|
|
"is_executing": false,
|
|
"name": "#%%\n"
|
|
}
|
|
},
|
|
"outputs": [
|
|
{
|
|
"name": "stdout",
|
|
"output_type": "stream",
|
|
"text": [
|
|
"[ { 'answer': 'Eddard',\n",
|
|
" 'context': 'ry warrior queen. She travels with her father, Eddard, to '\n",
|
|
" \"King's Landing when he is made Hand of the\"},\n",
|
|
" { 'answer': 'Ned',\n",
|
|
" 'context': '\\n'\n",
|
|
" '====Season 1====\\n'\n",
|
|
" 'Arya accompanies her father Ned and her sister Sansa to '\n",
|
|
" \"King's Landing. Before the\"},\n",
|
|
" { 'answer': 'Lord Eddard',\n",
|
|
" 'context': ' is the younger daughter and third child of Lord Eddard '\n",
|
|
" 'and Catelyn Stark of Winterfell. Ever the to'},\n",
|
|
" { 'answer': 'Lord Eddard Stark',\n",
|
|
" 'context': ' Tourney of the Hand to honour her father Lord Eddard '\n",
|
|
" 'Stark, Sansa Stark is enchanted by the knights'},\n",
|
|
" { 'answer': 'Eddard and Catelyn Stark',\n",
|
|
" 'context': 'e third child and younger daughter of Eddard and Catelyn '\n",
|
|
" 'Stark. She serves as a POV character for 33'}]\n"
|
|
]
|
|
}
|
|
],
|
|
"source": [
|
|
"print_answers(prediction, details=\"minimal\")"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.7.4"
|
|
},
|
|
"pycharm": {
|
|
"stem_cell": {
|
|
"cell_type": "raw",
|
|
"source": [],
|
|
"metadata": {
|
|
"collapsed": false
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
} |