mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-08-31 20:03:38 +00:00
Fetch QA model from remote in tutorial notebook
This commit is contained in:
Tanay Soni
parent
72f1ea32af
commit
6bc228fa6a
@ -19,17 +19,9 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Current working directory is /home/mp/deepset/dev/haystack\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Let's start by adjust the working directory so that it is the root of the repository\n",
|
||||
"# This should be run just once.\n",
|
||||
@ -40,22 +32,13 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"is_executing": false
|
||||
}
|
||||
},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"I1125 16:55:41.544814 139975239116608 file_utils.py:39] PyTorch version 1.3.0 available.\n",
|
||||
"I1125 16:55:41.619155 139975239116608 modeling_xlnet.py:194] Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from haystack.reader.farm import FARMReader\n",
|
||||
"from haystack.retriever.tfidf import TfidfRetriever\n",
|
||||
@ -85,9 +68,9 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"I1125 16:55:41.863932 139975239116608 io.py:57] Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip to `data/article_txt_got`\n",
|
||||
"100%|██████████| 1167348/1167348 [00:00<00:00, 9196388.44B/s]\n",
|
||||
"I1125 16:55:47.962270 139975239116608 io.py:30] Wrote 517 docs to DB\n"
|
||||
"11/28/2019 12:02:51 - INFO - haystack.indexing.io - Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip to `data/article_txt_got`\n",
|
||||
"100%|██████████| 1167348/1167348 [00:00<00:00, 8157729.91B/s]\n",
|
||||
"11/28/2019 12:02:52 - INFO - haystack.indexing.io - Wrote 517 docs to DB\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -129,7 +112,7 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"I1125 16:55:48.018222 139975239116608 tfidf.py:69] Found 2811 candidate paragraphs from 517 docs in DB\n"
|
||||
"11/28/2019 12:02:56 - INFO - haystack.retriever.tfidf - Found 2813 candidate paragraphs from 519 docs in DB\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -152,14 +135,18 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"W1125 16:55:53.613250 139975239116608 processor.py:170] Loading tokenizer from deprecated FARM config. If you used `custom_vocab` or `never_split_chars`, this won't work anymore.\n"
|
||||
"11/28/2019 12:02:57 - INFO - haystack.indexing.io - Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-models/0.3.0/bert-english-qa-large.tar.gz to `model`\n",
|
||||
"100%|██████████| 1245547135/1245547135 [00:53<00:00, 23191957.12B/s]\n",
|
||||
"11/28/2019 12:04:05 - WARNING - farm.data_handler.processor - Loading tokenizer from deprecated FARM config. If you used `custom_vocab` or `never_split_chars`, this won't work anymore.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# A reader scans the text chunks in detail and extracts the k best answers\n",
|
||||
"# Reader use more powerful but slower deep learning models, here: a BERT QA model trained via FARM on Squad 2.0\n",
|
||||
"reader = FARMReader(model_dir=\"../FARM/saved_models/bert-english-qa-large\", use_gpu=False)"
|
||||
"from haystack.indexing.io import fetch_archive_from_http\n",
|
||||
"fetch_archive_from_http(url=\"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-models/0.3.0/bert-english-qa-large.tar.gz\", output_dir=\"model\")\n",
|
||||
"reader = FARMReader(model_dir=\"model/bert-english-qa-large\", use_gpu=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@ -196,20 +183,20 @@
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"I1125 16:55:54.057870 139975239116608 __init__.py:92] Identified 10 candidates via retriever:\n",
|
||||
"11/28/2019 12:04:26 - INFO - haystack - Identified 10 candidates via retriever:\n",
|
||||
" paragraph_id document_id text\n",
|
||||
" 1257 227 \\n===Arya Stark===\\n'''Arya Stark''' portrayed by Maisie Williams. Arya Star...\n",
|
||||
" 1023 169 \\n====Season 8====\\nArya reunites with Jon, Gendry, and the Hound, who have ...\n",
|
||||
" 1016 169 \\n====Season 1====\\nArya accompanies her father Ned and her sister Sansa to ...\n",
|
||||
" 718 144 \\n===''A Game of Thrones''===\\nSansa Stark begins the novel by being betroth...\n",
|
||||
" 161 33 \\n===In Braavos===\\nLady Crane returns to her chambers to find a wounded Ary...\n",
|
||||
" 1846 304 \\n== Characters ==\\nThe tale is told through the eyes of 9 recurring POV cha...\n",
|
||||
" 1009 169 \\n==== ''A Game of Thrones'' ====\\nArya adopts a direwolf cub, which she nam...\n",
|
||||
" 1022 169 \\n====Season 7====\\nTaking the face of Walder Frey, Arya gathers the men of ...\n",
|
||||
" 847 163 \\n=== Arya Stark ===\\nArya Stark is the third child and younger daughter of ...\n",
|
||||
" 562 117 \\n===On the Kingsroad===\\nCity Watchmen search the caravan for Gendry but ar...\n",
|
||||
"I1125 16:55:54.058521 139975239116608 __init__.py:95] Applying the reader now to look for the answer in detail ...\n",
|
||||
"Inferencing: 100%|██████████| 1/1 [00:24<00:00, 24.35s/it]\n"
|
||||
" 2723 506 \\n===Arya Stark===\\n'''Arya Stark''' portrayed by Maisie Williams. Arya Star...\n",
|
||||
" 2212 407 \\n====Season 8====\\nArya reunites with Jon, Gendry, and the Hound, who have ...\n",
|
||||
" 2205 407 \\n====Season 1====\\nArya accompanies her father Ned and her sister Sansa to ...\n",
|
||||
" 548 105 \\n===''A Game of Thrones''===\\nSansa Stark begins the novel by being betroth...\n",
|
||||
" 1437 258 \\n===In Braavos===\\nLady Crane returns to her chambers to find a wounded Ary...\n",
|
||||
" 462 92 \\n== Characters ==\\nThe tale is told through the eyes of 9 recurring POV cha...\n",
|
||||
" 2198 407 \\n==== ''A Game of Thrones'' ====\\nArya adopts a direwolf cub, which she nam...\n",
|
||||
" 2211 407 \\n====Season 7====\\nTaking the face of Walder Frey, Arya gathers the men of ...\n",
|
||||
" 570 106 \\n=== Arya Stark ===\\nArya Stark is the third child and younger daughter of ...\n",
|
||||
" 313 65 \\n===On the Kingsroad===\\nCity Watchmen search the caravan for Gendry but ar...\n",
|
||||
"11/28/2019 12:04:27 - INFO - haystack - Applying the reader now to look for the answer in detail ...\n",
|
||||
"Inferencing: 100%|██████████| 1/1 [00:21<00:00, 21.82s/it]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
@ -231,7 +218,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 8,
|
||||
"metadata": {
|
||||
"pycharm": {
|
||||
"is_executing": false,
|
||||
@ -266,13 +253,6 @@
|
||||
"source": [
|
||||
"print_answers(prediction, details=\"minimal\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
@ -291,18 +271,18 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.7.3"
|
||||
"version": "3.7.4"
|
||||
},
|
||||
"pycharm": {
|
||||
"stem_cell": {
|
||||
"cell_type": "raw",
|
||||
"source": [],
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
}
|
||||
},
|
||||
"source": []
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user