mirror of
https://github.com/deepset-ai/haystack.git
synced 2026-01-05 03:28:09 +00:00
bf71f03ff2
* release v1.3.0 and re-add Makefile * Update Documentation & Code Style * make BaseKnowledgeGraph abstract to remove it from the JSON schema * Logging paths for JSON schema generation * Add debug command in autoforma.yml * Typo * Update Documentation & Code Style * Fix schema path in CI * Update Documentation & Code Style * Remove debug statement from autoformat.yml * Reintroduce compatibility between 1.3.0 and 1.2.1rc0 schema Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: ZanSara <sarazanzo94@gmail.com>
88 lines
3.9 KiB
JSON
88 lines
3.9 KiB
JSON
{
|
|
"chart_type": "BarChart",
|
|
"title": "Retriever Performance",
|
|
"subtitle": "Time and Accuracy Benchmarks",
|
|
"description": "Comparison of the speed and accuracy of different DocumentStore / Retriever combinations on 100k documents. <b>Indexing speed</b> (in docs/sec) refers to how quickly Documents can be inserted into a DocumentStore. <b>Querying speed</b> (in queries/sec) refers to the speed at which the system returns relevant Documents when presented with a query.\n\nThe dataset used is Wikipedia, split into 100 word passages (from <a href='https://github.com/facebookresearch/DPR/blob/master/dpr/data/download_data.py'>here</a>)). \n\nFor querying, we use the Natural Questions development set in combination with the wiki passages. The Document Store is populated with the 100 word passages in which the answer spans occur (i.e. gold passages) as well as a random selection of 100 word passages in which the answer spans do not occur (i.e. negative passages). We take a total of 100k gold and negative passages. Query and document embedding are generated by the <i>\"facebook/dpr-question_encoder-single-nq-base\"</i> and <i>\"facebook/dpr-ctx_encoder-single-nq-base\"</i> models. The retriever returns 10 candidates and both the recall and mAP scores are calculated on these 10.\n\nFor FAISS HNSW, we use <i>n_links=128</i>, <i>efSearch=20</i> and <i>efConstruction=80</i>. We use a cosine similarity function with BM25 retrievers, and dot product with DPR. Both index and query benchmarks are performed on an AWS P3.2xlarge instance which is accelerated by an Nvidia V100 GPU.",
|
|
"bars": "horizontal",
|
|
"columns": [
|
|
"Model",
|
|
"mAP",
|
|
"Index Speed (docs/sec)",
|
|
"Query Speed (queries/sec)"
|
|
],
|
|
"series": {
|
|
"s0": "map",
|
|
"s1": "time",
|
|
"s2": "time"
|
|
},
|
|
"axes": {
|
|
"label": "map",
|
|
"time_side": "top",
|
|
"time_label": "seconds"
|
|
},
|
|
"data": [
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"index_speed": 71.36964873196698,
|
|
"query_speed": 5.192368815242574,
|
|
"map": 86.54606328368976
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"index_speed": 485.5602670200369,
|
|
"query_speed": 103.0884393334727,
|
|
"map": 56.25299537353825
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"index_speed": 119.52937722555107,
|
|
"query_speed": 6.385621466857457,
|
|
"map": 82.74686664920836
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 100000,
|
|
"index_speed": 100.01184910084558,
|
|
"query_speed": 6.6270933964840415,
|
|
"map": 86.54606328368973
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 100000,
|
|
"index_speed": 89.90389306648805,
|
|
"query_speed": 39.7839528511866,
|
|
"map": 84.33419639513305
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (flat)",
|
|
"n_docs": 100000,
|
|
"index_speed": 116.00982709720004,
|
|
"query_speed": 28.57264344960955,
|
|
"map": 86.54606328368973
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (HNSW)",
|
|
"n_docs": 100000,
|
|
"index_speed": 115.61076852516383,
|
|
"query_speed": 38.80526238789059,
|
|
"map": 81.63864883662649
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"index_speed": 70.05381128388427,
|
|
"query_speed": 15.306895223372484,
|
|
"map": 86.54014997282701
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 100000,
|
|
"index_speed": 70.31004397719536,
|
|
"query_speed": 24.95733865947408,
|
|
"map": 85.7342431384476
|
|
}
|
|
]
|
|
} |