mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-12-29 16:08:38 +00:00
7e79a48540
* quick fix benchmark runs to make them work with current haystack version * fix minor typo * update readme. fix minor things to make benchmarks run again * Update Documentation & Code Style * fix typo in readme * update result files for reader and retriever querying * reduce batch size for update embeddings to prevent xlarge bulk_update requests that exceed elastic's limits (happening in dense 500k runs) * change default memory allocation back to normal. add note to readme * add first indexing results * add memory to docker cmd * full benchmarks results on commit c5a2651fcbbeffca06ffa9036b10e62669bcc1b0 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
184 lines
5.0 KiB
JSON
184 lines
5.0 KiB
JSON
{
|
|
"chart_type": "LineChart",
|
|
"title": "Retriever Accuracy",
|
|
"subtitle": "mAP at different number of docs",
|
|
"description": "Here you can see how the mean avg. precision (mAP) of the retriever decays as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
|
|
"columns": [
|
|
"n_docs",
|
|
"BM25 / Elasticsearch",
|
|
"DPR / Elasticsearch",
|
|
"DPR / FAISS (flat)",
|
|
"DPR / FAISS (HNSW)",
|
|
"DPR / Milvus (flat)",
|
|
"DPR / Milvus (HNSW)",
|
|
"Sentence Transformers / Elasticsearch"
|
|
],
|
|
"axis": [
|
|
{
|
|
"x": "Number of docs",
|
|
"y": "mAP"
|
|
}
|
|
],
|
|
"data": [
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"map": 76.49564526892908
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"map": 80.86137228234092
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"map": 90.06638620360428
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"map": 87.11255142468552
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"map": 86.5456409043424
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"map": 74.2044471297291
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"map": 45.603960228760656
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"map": 56.26106985872818
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"map": 66.32941083712781
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"map": 92.9510532283089
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904354
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"map": 82.74686664920834
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 500000,
|
|
"map": 80.86137228234091
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904354
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 1000,
|
|
"map": 92.9510532283089
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 100000,
|
|
"map": 86.54606328368972
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 100000,
|
|
"map": 84.33419639513305
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 10000,
|
|
"map": 89.51337675393017
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 1000,
|
|
"map": 92.9510532283089
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 500000,
|
|
"map": 75.73315903145605
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 10000,
|
|
"map": 66.32941083712774
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904354
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"map": 56.26106985872818
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 1000,
|
|
"map": 74.2044471297291
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"map": 86.51945338480733
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 500000,
|
|
"map": 45.59509026246651
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 500000,
|
|
"map": 80.861372282341
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 1000,
|
|
"map": 92.9510532283089
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904357
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 1000,
|
|
"map": 92.9510532283089
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 100000,
|
|
"map": 86.53719331739565
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 500000,
|
|
"map": 80.861372282341
|
|
}
|
|
]
|
|
} |