mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-12-29 16:08:38 +00:00
363be65a78
* Simplify ODES init * Add arguments to ES init and create script * Rename similarity_fn_name and add util fn * Create OpenSearchDocumentStore * Specify params of Open Search HNSW * Add better argument handling * Update opensearch index mapping * Edit opensearch default port * Fix HNSW mapping * Force small HNSW params * Implement auto start and stopping of document store services * Fix starting and stopping of ds service * Restore HNSW params * Add opensearch query benchmarks * Add write wait time * Revert wait time * Add timeout * Update benchmarks * Update benchmarks * Update benchmarks json * Update documentation * Update documentation * Fix similarity name * Improve argument passing * Improve stopping and starting of service
204 lines
5.5 KiB
JSON
204 lines
5.5 KiB
JSON
{
|
|
"chart_type": "LineChart",
|
|
"title": "Retriever Accuracy",
|
|
"subtitle": "mAP at different number of docs",
|
|
"description": "Here you can see how the mean avg. precision (mAP) of the retriever decays as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
|
|
"columns": [
|
|
"n_docs",
|
|
"BM25 / Elasticsearch",
|
|
"DPR / Elasticsearch",
|
|
"DPR / FAISS (flat)",
|
|
"DPR / FAISS (HNSW)",
|
|
"DPR / Milvus (flat)",
|
|
"DPR / Milvus (HNSW)",
|
|
"Sentence Transformers / Elasticsearch"
|
|
],
|
|
"axis": [
|
|
{
|
|
"x": "Number of docs",
|
|
"y": "mAP"
|
|
}
|
|
],
|
|
"data": [
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"map": 92.95105322830891
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904354
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"map": 66.26543444531747
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"map": 90.06638620360428
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"map": 87.11255142468549
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 1000,
|
|
"map": 92.95105322830891
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904354
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 1000,
|
|
"map": 92.95105322830891
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 10000,
|
|
"map": 89.51337675393017
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (flat)",
|
|
"n_docs": 1000,
|
|
"map": 92.95105322830891
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (flat)",
|
|
"n_docs": 10000,
|
|
"map": 89.87097014904354
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (HNSW)",
|
|
"n_docs": 1000,
|
|
"map": 92.95105322830891
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (HNSW)",
|
|
"n_docs": 10000,
|
|
"map": 88.24421129104469
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"map": 86.54606328368976
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"map": 80.86137228234091
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"map": 56.25299537353825
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"map": 45.595090262466535
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"map": 82.74686664920836
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"map": 76.49564526892904
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 100000,
|
|
"map": 86.54606328368973
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 500000,
|
|
"map": 80.86137228234091
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 100000,
|
|
"map": 84.33419639513305
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 500000,
|
|
"map": 75.73062475537202
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (flat)",
|
|
"n_docs": 100000,
|
|
"map": 86.54606328368973
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (flat)",
|
|
"n_docs": 500000,
|
|
"map": 80.86137228234091
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (HNSW)",
|
|
"n_docs": 100000,
|
|
"map": 81.63864883662649
|
|
},
|
|
{
|
|
"model": "DPR / Milvus (HNSW)",
|
|
"n_docs": 500000,
|
|
"map": 73.57986207906387
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"map": 74.20444712972909
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 1000,
|
|
"map": 92.95105322830891
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 10000,
|
|
"map": 89.8709701490436
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"map": 86.54014997282701
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 1000,
|
|
"map": 92.76308330349686
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 10000,
|
|
"map": 89.00403653862938
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 100000,
|
|
"map": 85.7342431384476
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 500000,
|
|
"map": 80.85588135082547
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 500000,
|
|
"map": 77.5426462347698
|
|
}
|
|
]
|
|
} |