From 6bc228fa6accebc2ef1ba8895bc15c4a0c2833e8 Mon Sep 17 00:00:00 2001 From: Tanay Soni Date: Thu, 28 Nov 2019 12:07:04 +0100 Subject: [PATCH] Fetch QA model from remote in tutorial notebook --- tutorials/Tutorial1_Basic_QA_Pipeline.ipynb | 84 ++++++++------------- 1 file changed, 32 insertions(+), 52 deletions(-) diff --git a/tutorials/Tutorial1_Basic_QA_Pipeline.ipynb b/tutorials/Tutorial1_Basic_QA_Pipeline.ipynb index f88a000f5..82fb41734 100644 --- a/tutorials/Tutorial1_Basic_QA_Pipeline.ipynb +++ b/tutorials/Tutorial1_Basic_QA_Pipeline.ipynb @@ -19,17 +19,9 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Current working directory is /home/mp/deepset/dev/haystack\n" - ] - } - ], + "outputs": [], "source": [ "# Let's start by adjust the working directory so that it is the root of the repository\n", "# This should be run just once.\n", @@ -40,22 +32,13 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "pycharm": { "is_executing": false } }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "I1125 16:55:41.544814 139975239116608 file_utils.py:39] PyTorch version 1.3.0 available.\n", - "I1125 16:55:41.619155 139975239116608 modeling_xlnet.py:194] Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex .\n" - ] - } - ], + "outputs": [], "source": [ "from haystack.reader.farm import FARMReader\n", "from haystack.retriever.tfidf import TfidfRetriever\n", @@ -85,9 +68,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "I1125 16:55:41.863932 139975239116608 io.py:57] Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip to `data/article_txt_got`\n", - "100%|██████████| 1167348/1167348 [00:00<00:00, 9196388.44B/s]\n", - "I1125 16:55:47.962270 139975239116608 io.py:30] Wrote 517 docs to DB\n" + "11/28/2019 12:02:51 - INFO - haystack.indexing.io - Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/documents/wiki_gameofthrones_txt.zip to `data/article_txt_got`\n", + "100%|██████████| 1167348/1167348 [00:00<00:00, 8157729.91B/s]\n", + "11/28/2019 12:02:52 - INFO - haystack.indexing.io - Wrote 517 docs to DB\n" ] } ], @@ -129,7 +112,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "I1125 16:55:48.018222 139975239116608 tfidf.py:69] Found 2811 candidate paragraphs from 517 docs in DB\n" + "11/28/2019 12:02:56 - INFO - haystack.retriever.tfidf - Found 2813 candidate paragraphs from 519 docs in DB\n" ] } ], @@ -152,14 +135,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "W1125 16:55:53.613250 139975239116608 processor.py:170] Loading tokenizer from deprecated FARM config. If you used `custom_vocab` or `never_split_chars`, this won't work anymore.\n" + "11/28/2019 12:02:57 - INFO - haystack.indexing.io - Fetching from https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-models/0.3.0/bert-english-qa-large.tar.gz to `model`\n", + "100%|██████████| 1245547135/1245547135 [00:53<00:00, 23191957.12B/s]\n", + "11/28/2019 12:04:05 - WARNING - farm.data_handler.processor - Loading tokenizer from deprecated FARM config. If you used `custom_vocab` or `never_split_chars`, this won't work anymore.\n" ] } ], "source": [ "# A reader scans the text chunks in detail and extracts the k best answers\n", "# Reader use more powerful but slower deep learning models, here: a BERT QA model trained via FARM on Squad 2.0\n", - "reader = FARMReader(model_dir=\"../FARM/saved_models/bert-english-qa-large\", use_gpu=False)" + "from haystack.indexing.io import fetch_archive_from_http\n", + "fetch_archive_from_http(url=\"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-models/0.3.0/bert-english-qa-large.tar.gz\", output_dir=\"model\")\n", + "reader = FARMReader(model_dir=\"model/bert-english-qa-large\", use_gpu=False)" ] }, { @@ -196,20 +183,20 @@ "name": "stderr", "output_type": "stream", "text": [ - "I1125 16:55:54.057870 139975239116608 __init__.py:92] Identified 10 candidates via retriever:\n", + "11/28/2019 12:04:26 - INFO - haystack - Identified 10 candidates via retriever:\n", " paragraph_id document_id text\n", - " 1257 227 \\n===Arya Stark===\\n'''Arya Stark''' portrayed by Maisie Williams. Arya Star...\n", - " 1023 169 \\n====Season 8====\\nArya reunites with Jon, Gendry, and the Hound, who have ...\n", - " 1016 169 \\n====Season 1====\\nArya accompanies her father Ned and her sister Sansa to ...\n", - " 718 144 \\n===''A Game of Thrones''===\\nSansa Stark begins the novel by being betroth...\n", - " 161 33 \\n===In Braavos===\\nLady Crane returns to her chambers to find a wounded Ary...\n", - " 1846 304 \\n== Characters ==\\nThe tale is told through the eyes of 9 recurring POV cha...\n", - " 1009 169 \\n==== ''A Game of Thrones'' ====\\nArya adopts a direwolf cub, which she nam...\n", - " 1022 169 \\n====Season 7====\\nTaking the face of Walder Frey, Arya gathers the men of ...\n", - " 847 163 \\n=== Arya Stark ===\\nArya Stark is the third child and younger daughter of ...\n", - " 562 117 \\n===On the Kingsroad===\\nCity Watchmen search the caravan for Gendry but ar...\n", - "I1125 16:55:54.058521 139975239116608 __init__.py:95] Applying the reader now to look for the answer in detail ...\n", - "Inferencing: 100%|██████████| 1/1 [00:24<00:00, 24.35s/it]\n" + " 2723 506 \\n===Arya Stark===\\n'''Arya Stark''' portrayed by Maisie Williams. Arya Star...\n", + " 2212 407 \\n====Season 8====\\nArya reunites with Jon, Gendry, and the Hound, who have ...\n", + " 2205 407 \\n====Season 1====\\nArya accompanies her father Ned and her sister Sansa to ...\n", + " 548 105 \\n===''A Game of Thrones''===\\nSansa Stark begins the novel by being betroth...\n", + " 1437 258 \\n===In Braavos===\\nLady Crane returns to her chambers to find a wounded Ary...\n", + " 462 92 \\n== Characters ==\\nThe tale is told through the eyes of 9 recurring POV cha...\n", + " 2198 407 \\n==== ''A Game of Thrones'' ====\\nArya adopts a direwolf cub, which she nam...\n", + " 2211 407 \\n====Season 7====\\nTaking the face of Walder Frey, Arya gathers the men of ...\n", + " 570 106 \\n=== Arya Stark ===\\nArya Stark is the third child and younger daughter of ...\n", + " 313 65 \\n===On the Kingsroad===\\nCity Watchmen search the caravan for Gendry but ar...\n", + "11/28/2019 12:04:27 - INFO - haystack - Applying the reader now to look for the answer in detail ...\n", + "Inferencing: 100%|██████████| 1/1 [00:21<00:00, 21.82s/it]\n" ] } ], @@ -231,7 +218,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "pycharm": { "is_executing": false, @@ -266,13 +253,6 @@ "source": [ "print_answers(prediction, details=\"minimal\")" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -291,18 +271,18 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.4" }, "pycharm": { "stem_cell": { "cell_type": "raw", - "source": [], "metadata": { "collapsed": false - } + }, + "source": [] } } }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +}