yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-09-26 08:33:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

update eval dataset

Browse Source
This commit is contained in:
Branden Chan 2020-07-15 16:14:52 +02:00
parent 912e98cd40
commit c55477e0ce
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tutorials/Tutorial5_Evaluation.ipynb
View File

@ -1639,7 +1639,7 @@
"\n", "\n",
"# Download evaluation data, which is a subset of Natural Questions development set containing 50 documents\n", "# Download evaluation data, which is a subset of Natural Questions development set containing 50 documents\n",
"doc_dir = \"../data/nq\"\n", "doc_dir = \"../data/nq\"\n",
"s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset.json.zip\"\n", "s3_url = \"https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip\"\n",
"fetch_archive_from_http(url=s3_url, output_dir=doc_dir)" "fetch_archive_from_http(url=s3_url, output_dir=doc_dir)"
], ],
"execution_count": 7, "execution_count": 7,
@ -1701,7 +1701,7 @@
}, },
"source": [ "source": [
"# Add evaluation data to Elasticsearch database\n", "# Add evaluation data to Elasticsearch database\n",
"document_store.add_eval_data(\"../data/nq/nq_dev_subset.json\")" "document_store.add_eval_data(\"../data/nq/nq_dev_subset_v2.json\")"
], ],
"execution_count": 9, "execution_count": 9,
"outputs": [ "outputs": [
@ -2283,7 +2283,7 @@
"\n", "\n",
"# Evaluation of Reader can also be done directly on a SQuAD-formatted file\n", "# Evaluation of Reader can also be done directly on a SQuAD-formatted file\n",
"# without passing the data to Elasticsearch\n", "# without passing the data to Elasticsearch\n",
"#reader_eval_results = reader.eval_on_file(\"../data/natural_questions\", \"dev_subset.json\", device=device)\n", "#reader_eval_results = reader.eval_on_file(\"../data/natural_questions\", \"dev_subset_v2.json\", device=device)\n",
"\n", "\n",
"## Reader Top-N-Recall is the proportion of predicted answers that overlap with their corresponding correct answer\n", "## Reader Top-N-Recall is the proportion of predicted answers that overlap with their corresponding correct answer\n",
"print(\"Reader Top-N-Recall:\", reader_eval_results[\"top_n_recall\"])\n", "print(\"Reader Top-N-Recall:\", reader_eval_results[\"top_n_recall\"])\n",

6
tutorials/Tutorial5_Evaluation.py
View File

@ -39,17 +39,17 @@ if LAUNCH_ELASTICSEARCH:
# Download evaluation data, which is a subset of Natural Questions development set containing 50 documents # Download evaluation data, which is a subset of Natural Questions development set containing 50 documents
doc_dir = "../data/nq" doc_dir = "../data/nq"
s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset.json.zip" s3_url = "https://s3.eu-central-1.amazonaws.com/deepset.ai-farm-qa/datasets/nq_dev_subset_v2.json.zip"
fetch_archive_from_http(url=s3_url, output_dir=doc_dir) fetch_archive_from_http(url=s3_url, output_dir=doc_dir)
# Connect to Elasticsearch # Connect to Elasticsearch
document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document", create_index=False) document_store = ElasticsearchDocumentStore(host="localhost", username="", password="", index="document", create_index=False)
# Add evaluation data to Elasticsearch database # Add evaluation data to Elasticsearch database
if LAUNCH_ELASTICSEARCH: if LAUNCH_ELASTICSEARCH:
document_store.add_eval_data("../data/nq/nq_dev_subset.json") document_store.add_eval_data("../data/nq/nq_dev_subset_v2.json")
else: else:
logger.warning("Since we already have a running ES instance we should not index the same documents again." logger.warning("Since we already have a running ES instance we should not index the same documents again."
"If you still want to do this call: 'document_store.add_eval_data('../data/nq/nq_dev_subset.json')' manually ") "If you still want to do this call: 'document_store.add_eval_data('../data/nq/nq_dev_subset_v2.json')' manually ")
# Initialize Retriever # Initialize Retriever
retriever = ElasticsearchRetriever(document_store=document_store) retriever = ElasticsearchRetriever(document_store=document_store)