mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-12-28 07:29:06 +00:00
2531c8e061
* add time and perf benchmark for es * Add retriever benchmarking * Add Reader benchmarking * add nq to squad conversion * add conversion stats * clean benchmarks * Add link to dataset * Update imports * add first support for neg psgs * Refactor test * set max_seq_len * cleanup benchmark * begin retriever speed benchmarking * Add support for retriever query index benchmarking * improve reader eval, retriever speed benchmarking * improve retriever speed benchmarking * Add retriever accuracy benchmark * Add neg doc shuffling * Add top_n * 3x speedup of SQL. add postgres docker run. make shuffle neg a param. add more logging * Add models to sweep * add option for faiss index type * remove unneeded line * change faiss to faiss_flat * begin automatic benchmark script * remove existing postgres docker for benchmarking * Add data processing scripts * Remove shuffle in script bc data already shuffled * switch hnsw setup from 256 to 128 * change es similarity to dot product by default * Error includes stack trace * Change ES default timeout * remove delete_docs() from timing for indexing * Add support for website export * update website on push to benchmarks * add complete benchmarks results * new json format * removed NaN as is not a valid json token * versioning for docs * unsaved changes * cleaning * cleaning * Edit format of benchmarks data * update also jsons in v0.4.0 Co-authored-by: brandenchan <brandenchan@icloud.com> Co-authored-by: deepset <deepset@Crenolape.localdomain> Co-authored-by: Malte Pietsch <malte.pietsch@deepset.ai>
99 lines
2.7 KiB
JSON
99 lines
2.7 KiB
JSON
{
|
|
"chart_type": "LineChart",
|
|
"title": "Retriever Speed",
|
|
"subtitle": "Query Speed at different number of docs",
|
|
"description": "Here you can see how the query speed of different Retriever / DocumentStore combinations scale as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
|
|
"columns": [
|
|
"n_docs",
|
|
"BM25 / ElasticSearch",
|
|
"DPR / ElasticSearch",
|
|
"DPR / FAISS (flat)",
|
|
"DPR / FAISS (HSNW)"
|
|
],
|
|
"axis": [
|
|
{ "x": "Number of docs", "y": "Docs/sec" }
|
|
],
|
|
"data":
|
|
[
|
|
{
|
|
"model": "DPR / ElasticSearch",
|
|
"n_docs": 1000,
|
|
"query_speed": 40.802
|
|
},
|
|
{
|
|
"model": "DPR / ElasticSearch",
|
|
"n_docs": 10000,
|
|
"query_speed": 27.006999999999998
|
|
},
|
|
{
|
|
"model": "DPR / ElasticSearch",
|
|
"n_docs": 100000,
|
|
"query_speed": 6.5360000000000005
|
|
},
|
|
{
|
|
"model": "DPR / ElasticSearch",
|
|
"n_docs": 500000,
|
|
"query_speed": 1.514
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 1000,
|
|
"query_speed": 40.048
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 10000,
|
|
"query_speed": 23.976999999999997
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 100000,
|
|
"query_speed": 5.044
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 500000,
|
|
"query_speed": 1.091
|
|
},
|
|
{
|
|
"model": "BM25 / ElasticSearch",
|
|
"n_docs": 1000,
|
|
"query_speed": 232.97799999999998
|
|
},
|
|
{
|
|
"model": "BM25 / ElasticSearch",
|
|
"n_docs": 10000,
|
|
"query_speed": 167.81
|
|
},
|
|
{
|
|
"model": "BM25 / ElasticSearch",
|
|
"n_docs": 100000,
|
|
"query_speed": 162.996
|
|
},
|
|
{
|
|
"model": "BM25 / ElasticSearch",
|
|
"n_docs": 500000,
|
|
"query_speed": 95.491
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HSNW)",
|
|
"n_docs": 1000,
|
|
"query_speed": 37.884
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HSNW)",
|
|
"n_docs": 10000,
|
|
"query_speed": 33.421
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HSNW)",
|
|
"n_docs": 100000,
|
|
"query_speed": 12.815
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HSNW)",
|
|
"n_docs": 500000,
|
|
"query_speed": 3.259
|
|
}
|
|
]
|
|
} |