mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-12-31 09:10:15 +00:00
7e79a48540
* quick fix benchmark runs to make them work with current haystack version * fix minor typo * update readme. fix minor things to make benchmarks run again * Update Documentation & Code Style * fix typo in readme * update result files for reader and retriever querying * reduce batch size for update embeddings to prevent xlarge bulk_update requests that exceed elastic's limits (happening in dense 500k runs) * change default memory allocation back to normal. add note to readme * add first indexing results * add memory to docker cmd * full benchmarks results on commit c5a2651fcbbeffca06ffa9036b10e62669bcc1b0 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
184 lines
5.3 KiB
JSON
184 lines
5.3 KiB
JSON
{
|
|
"chart_type": "LineChart",
|
|
"title": "Retriever Speed",
|
|
"subtitle": "Query Speed at different number of docs",
|
|
"description": "Here you can see how the query speed of different Retriever / DocumentStore combinations scale as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
|
|
"columns": [
|
|
"n_docs",
|
|
"BM25 / Elasticsearch",
|
|
"DPR / Elasticsearch",
|
|
"DPR / FAISS (flat)",
|
|
"DPR / FAISS (HNSW)",
|
|
"DPR / Milvus (flat)",
|
|
"DPR / Milvus (HNSW)",
|
|
"Sentence Transformers / Elasticsearch"
|
|
],
|
|
"axis": [
|
|
{
|
|
"x": "Number of docs",
|
|
"y": "Queries/sec"
|
|
}
|
|
],
|
|
"data": [
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"query_speed": 1.4416146543448518
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"query_speed": 1.4681488854743858
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"query_speed": 48.176051675516526
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"query_speed": 30.0778559137467
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"query_speed": 6.239008635095737
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"query_speed": 258.12517634164914
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 500000,
|
|
"query_speed": 67.45949681882962
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"query_speed": 132.02605052193084
|
|
},
|
|
{
|
|
"model": "BM25 / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"query_speed": 196.51035140878585
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 1000,
|
|
"query_speed": 36.06325436233689
|
|
},
|
|
{
|
|
"model": "DPR / Elasticsearch",
|
|
"n_docs": 10000,
|
|
"query_speed": 25.07636705997116
|
|
},
|
|
{
|
|
"model": "Sentence Transformers / Elasticsearch",
|
|
"n_docs": 100000,
|
|
"query_speed": 6.358106794217024
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 500000,
|
|
"query_speed": 1.5303310709494675
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 10000,
|
|
"query_speed": 31.06388073640026
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 1000,
|
|
"query_speed": 35.49036694594937
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (flat)",
|
|
"n_docs": 100000,
|
|
"query_speed": 6.700916380795566
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 100000,
|
|
"query_speed": 40.6580803190388
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 10000,
|
|
"query_speed": 50.01305495228924
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 1000,
|
|
"query_speed": 41.538171283159
|
|
},
|
|
{
|
|
"model": "DPR / FAISS (HNSW)",
|
|
"n_docs": 500000,
|
|
"query_speed": 45.097511850660666
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 10000,
|
|
"query_speed": 173.8232982275283
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 10000,
|
|
"query_speed": 30.86623936563611
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"query_speed": 120.12526583899732
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 1000,
|
|
"query_speed": 174.3733594241761
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 100000,
|
|
"query_speed": 17.604366772353018
|
|
},
|
|
{
|
|
"model": "BM25 / OpenSearch (flat)",
|
|
"n_docs": 500000,
|
|
"query_speed": 67.59304756122785
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 500000,
|
|
"query_speed": 12.09249157007648
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (flat)",
|
|
"n_docs": 1000,
|
|
"query_speed": 35.35562598951788
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 10000,
|
|
"query_speed": 30.986629506879076
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 1000,
|
|
"query_speed": 35.67303735213834
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 100000,
|
|
"query_speed": 17.226750993345046
|
|
},
|
|
{
|
|
"model": "DPR / OpenSearch (HNSW)",
|
|
"n_docs": 500000,
|
|
"query_speed": 11.809321812785312
|
|
}
|
|
]
|
|
} |