mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-12-30 00:30:09 +00:00
7e79a48540
* quick fix benchmark runs to make them work with current haystack version * fix minor typo * update readme. fix minor things to make benchmarks run again * Update Documentation & Code Style * fix typo in readme * update result files for reader and retriever querying * reduce batch size for update embeddings to prevent xlarge bulk_update requests that exceed elastic's limits (happening in dense 500k runs) * change default memory allocation back to normal. add note to readme * add first indexing results * add memory to docker cmd * full benchmarks results on commit c5a2651fcbbeffca06ffa9036b10e62669bcc1b0 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
81 lines
3.7 KiB
JSON
81 lines
3.7 KiB
JSON
{
|
|
"chart_type": "BarChart",
|
|
"title": "Retriever Performance",
|
|
"subtitle": "Time and Accuracy Benchmarks",
|
|
"description": "Comparison of the speed and accuracy of different DocumentStore / Retriever combinations on 100k documents. <b>Indexing speed</b> (in docs/sec) refers to how quickly Documents can be inserted into a DocumentStore. <b>Querying speed</b> (in queries/sec) refers to the speed at which the system returns relevant Documents when presented with a query.\n\nThe dataset used is Wikipedia, split into 100 word passages (from <a href='https://github.com/facebookresearch/DPR/blob/master/dpr/data/download_data.py'>here</a>)). \n\nFor querying, we use the Natural Questions development set in combination with the wiki passages. The Document Store is populated with the 100 word passages in which the answer spans occur (i.e. gold passages) as well as a random selection of 100 word passages in which the answer spans do not occur (i.e. negative passages). We take a total of 100k gold and negative passages. Query and document embedding are generated by the <i>\"facebook/dpr-question_encoder-single-nq-base\"</i> and <i>\"facebook/dpr-ctx_encoder-single-nq-base\"</i> models. The retriever returns 10 candidates and both the recall and mAP scores are calculated on these 10.\n\nFor FAISS HNSW, we use <i>n_links=128</i>, <i>efSearch=20</i> and <i>efConstruction=80</i>. We use a cosine similarity function with BM25 retrievers, and dot product with DPR. Both index and query benchmarks are performed on an AWS P3.2xlarge instance which is accelerated by an Nvidia V100 GPU.",
|
|
"bars": "horizontal",
|
|
"columns": [
|
|
"Model",
|
|
"mAP",
|
|
"Index Speed (docs/sec)",
|
|
"Query Speed (queries/sec)"
|
|
],
|
|
"series": {
|
|
"s0": "map",
|
|
"s1": "time",
|
|
"s2": "time"
|
|
},
|
|
"axes": {
|
|
"label": "map",
|
|
"time_side": "top",
|
|
"time_label": "seconds"
|
|
},
|
|
"data": [
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"index_speed": 81.2504988059173,
|
|
"query_speed": 6.239008635095737,
|
|
"map": 86.5456409043424
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"index_speed": 483.4698964792531,
|
|
"query_speed": 132.02605052193084,
|
|
"map": 56.26106985872818
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"index_speed": 120.24128813537725,
|
|
"query_speed": 6.358106794217024,
|
|
"map": 82.74686664920834
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 100000,
|
|
"index_speed": 93.59054930302146,
|
|
"query_speed": 6.700916380795566,
|
|
"map": 86.54606328368972
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 100000,
|
|
"index_speed": 83.61910261947709,
|
|
"query_speed": 40.6580803190388,
|
|
"map": 84.33419639513305
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"index_speed": 478.7131568904452,
|
|
"query_speed": 120.12526583899732,
|
|
"map": 56.26106985872818
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"index_speed": 80.32081169498916,
|
|
"query_speed": 17.604366772353018,
|
|
"map": 86.51945338480733
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 100000,
|
|
"index_speed": 80.45309468360983,
|
|
"query_speed": 17.226750993345046,
|
|
"map": 86.53719331739565
|
|
}
|
|
]
|
|
} |