yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-10-14 17:38:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add latest benchmark run (#652)

Browse Source 
* add latest benchmark run

* update templates and fix small json errors

* Change scale

Co-authored-by: brandenchan <brandenchan@icloud.com>
This commit is contained in:
Malte Pietsch 2020-12-10 16:25:51 +01:00 committed by GitHub
parent efc754b166
commit 149d98a0fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 245 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
docs/_src/benchmarks/reader_performance.json
View File

@ -10,10 +10,35 @@
"Speed (passages/sec)" "Speed (passages/sec)"
], ],
"data": [ "data": [
{"F1": 80.67985794671885, "Model": "RoBERTa", "Speed": 92.3039712094936}, {
{"F1": 78.23306265318686, "Model": "MiniLM", "Speed": 98.62387044489223}, "F1": 82.62983412843887,
{"F1": 74.90271600053505, "Model": "BERT base", "Speed": 99.92750782409666}, "Speed": 98.86638639776464,
{"F1": 82.64545708097472, "Model": "BERT large", "Speed": 39.529824033964466}, "Model": "RoBERTa"
{"F1": 85.26275190954586, "Model": "XLM-RoBERTa", "Speed": 39.29142006004379} },
{
"F1": 78.90026641413856,
"Speed": 181.96379531485616,
"Model": "MiniLM"
},
{
"F1": 74.32668866064459,
"Speed": 106.04748306200683,
"Model": "BERT base"
},
{
"F1": 83.29492827667042,
"Speed": 40.408497243719076,
"Model": "BERT large"
},
{
"F1": 84.62174414643722,
"Speed": 40.483264542292716,
"Model": "XLM-RoBERTa"
},
{
"F1": 42.342513261953935,
"Speed": 160.41712955027901,
"Model": "DistilBERT"
}
] ]
} }

42
docs/_src/benchmarks/retriever_map.json
View File

@ -6,30 +6,54 @@
"columns": [ "columns": [
"n_docs", "n_docs",
"BM25 / ElasticSearch", "BM25 / ElasticSearch",
"DPR / ElasticSearch or FAISS (flat)", "DPR / ElasticSearch",
"DPR / FAISS (flat)",
"DPR / FAISS (HSNW)" "DPR / FAISS (HSNW)"
], ],
"axis": [ "axis": [
{ "x": "Number of docs", "y": "mAP" } {
"x": "Number of docs",
"y": "mAP"
}
], ],
"data": [ "data": [
{ {
"model": "DPR / ElasticSearch or FAISS (flat)", "model": "DPR / ElasticSearch",
"n_docs": 1000, "n_docs": 1000,
"map": 0.929 "map": 0.929
}, },
{ {
"model": "DPR / ElasticSearch or FAISS (flat)", "model": "DPR / ElasticSearch",
"n_docs": 10000,
"map": 0.881
},
{
"model": "DPR / ElasticSearch",
"n_docs": 100000,
"map": 0.821
},
{
"model": "DPR / ElasticSearch",
"n_docs": 500000,
"map": 0.730
},
{
"model": "DPR / FAISS (flat)",
"n_docs": 1000,
"map": 0.929
},
{
"model": "DPR / FAISS (flat)",
"n_docs": 10000, "n_docs": 10000,
"map": 0.898 "map": 0.898
}, },
{ {
"model": "DPR / ElasticSearch or FAISS (flat)", "model": "DPR / FAISS (flat)",
"n_docs": 100000, "n_docs": 100000,
"map": 0.863 "map": 0.863
}, },
{ {
"model": "DPR / ElasticSearch or FAISS (flat)", "model": "DPR / FAISS (flat)",
"n_docs": 500000, "n_docs": 500000,
"map": 0.805 "map": 0.805
}, },
@ -61,17 +85,17 @@
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 10000, "n_docs": 10000,
"map": 0.8959999999999999 "map": 0.896
}, },
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 100000, "n_docs": 100000,
"map": 0.8490000000000001 "map": 0.849
}, },
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 500000, "n_docs": 500000,
"map": 0.7659999999999999 "map": 0.766
} }
] ]
} }

21
docs/_src/benchmarks/retriever_performance.json
View File

@ -24,31 +24,30 @@
{ {
"model": "DPR / ElasticSearch", "model": "DPR / ElasticSearch",
"n_docs": 100000, "n_docs": 100000,
"index_speed": 79.54165185, "index_speed": 73.93635160290218,
"query_speed": 6.5360000000000005, "query_speed": 6.23,
"map": 86.3 "map": 82
}, },
{ {
"model": "DPR / FAISS (flat)", "model": "DPR / FAISS (flat)",
"n_docs": 100000, "n_docs": 100000,
"index_speed": 107.8662479, "index_speed": 104.77116699738369,
"query_speed": 5.044, "query_speed": 4.89,
"map": 86.3 "map": 86.3
}, },
{ {
"model": "BM25 / ElasticSearch", "model": "BM25 / ElasticSearch",
"n_docs": 100000, "n_docs": 100000,
"index_speed": 476.9143596, "index_speed": 484.32931514144724,
"query_speed": 162.996, "query_speed": 162.59,
"map": 56.0 "map": 56
}, },
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 100000, "n_docs": 100000,
"index_speed": 92.24548333, "index_speed": 91.41086878008392,
"query_speed": 12.815, "query_speed": 12.85,
"map": 84.9 "map": 84.9
} }
] ]
} }

32
docs/_src/benchmarks/retriever_speed.json
View File

@ -11,10 +11,12 @@
"DPR / FAISS (HSNW)" "DPR / FAISS (HSNW)"
], ],
"axis": [ "axis": [
{ "x": "Number of docs", "y": "Queries/sec" } {
"x": "Number of docs",
"y": "Queries/sec"
}
], ],
"data": "data": [
[
{ {
"model": "DPR / ElasticSearch", "model": "DPR / ElasticSearch",
"n_docs": 1000, "n_docs": 1000,
@ -23,17 +25,17 @@
{ {
"model": "DPR / ElasticSearch", "model": "DPR / ElasticSearch",
"n_docs": 10000, "n_docs": 10000,
"query_speed": 27.006999999999998 "query_speed": 24.8
}, },
{ {
"model": "DPR / ElasticSearch", "model": "DPR / ElasticSearch",
"n_docs": 100000, "n_docs": 100000,
"query_speed": 6.5360000000000005 "query_speed": 6.23
}, },
{ {
"model": "DPR / ElasticSearch", "model": "DPR / ElasticSearch",
"n_docs": 500000, "n_docs": 500000,
"query_speed": 1.514 "query_speed": 1.45
}, },
{ {
"model": "DPR / FAISS (flat)", "model": "DPR / FAISS (flat)",
@ -43,17 +45,17 @@
{ {
"model": "DPR / FAISS (flat)", "model": "DPR / FAISS (flat)",
"n_docs": 10000, "n_docs": 10000,
"query_speed": 23.976999999999997 "query_speed": 22.47
}, },
{ {
"model": "DPR / FAISS (flat)", "model": "DPR / FAISS (flat)",
"n_docs": 100000, "n_docs": 100000,
"query_speed": 5.044 "query_speed": 4.90
}, },
{ {
"model": "DPR / FAISS (flat)", "model": "DPR / FAISS (flat)",
"n_docs": 500000, "n_docs": 500000,
"query_speed": 1.091 "query_speed": 1.08
}, },
{ {
"model": "BM25 / ElasticSearch", "model": "BM25 / ElasticSearch",
@ -63,17 +65,17 @@
{ {
"model": "BM25 / ElasticSearch", "model": "BM25 / ElasticSearch",
"n_docs": 10000, "n_docs": 10000,
"query_speed": 167.81 "query_speed": 248.97
}, },
{ {
"model": "BM25 / ElasticSearch", "model": "BM25 / ElasticSearch",
"n_docs": 100000, "n_docs": 100000,
"query_speed": 162.996 "query_speed": 162.59
}, },
{ {
"model": "BM25 / ElasticSearch", "model": "BM25 / ElasticSearch",
"n_docs": 500000, "n_docs": 500000,
"query_speed": 95.491 "query_speed": 91.39
}, },
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
@ -83,17 +85,17 @@
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 10000, "n_docs": 10000,
"query_speed": 33.421 "query_speed": 31.34
}, },
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 100000, "n_docs": 100000,
"query_speed": 12.815 "query_speed": 12.85
}, },
{ {
"model": "DPR / FAISS (HSNW)", "model": "DPR / FAISS (HSNW)",
"n_docs": 500000, "n_docs": 500000,
"query_speed": 3.259 "query_speed": 3.32
} }
] ]
} }

11
test/benchmarks/reader_results.csv
View File

@ -1,6 +1,7 @@
,EM,f1,top_n_accuracy,top_n,reader_time,seconds_per_query,passages_per_second,reader,error ,EM,f1,top_n_accuracy,top_n,reader_time,seconds_per_query,passages_per_second,reader,error
0,0.7589752233271532,0.8067985794671885,0.9671329849991572,5,133.79706027999998,0.011275666634080564,92.30397120949361,deepset/roberta-base-squad2, 0,0.7836676217765043,0.8262983412843887,0.9742963087813922,5,124.91606550999859,0.01052722615118815,98.86638639776463,deepset/roberta-base-squad2,
1,0.7359683128265633,0.7823306265318686,0.9714309792684982,5,125.22323393199997,0.010553112584864317,98.62387044489225,deepset/minilm-uncased-squad2, 1,0.7439743805831789,0.7890026641413856,0.9720209000505646,5,67.87064415001078,0.005719757639475036,181.96379531485616,deepset/minilm-uncased-squad2,
2,0.700825889094893,0.7490271600053505,0.9585369964604753,5,123.58959278499992,0.010415438461570867,99.92750782409666,deepset/bert-base-cased-squad2, 2,0.6947581324793528,0.7432668866064459,0.9557559413450194,5,116.45726653200109,0.009814365964267747,106.04748306200683,deepset/bert-base-cased-squad2,
3,0.7821506826226192,0.8264545708097472,0.9762346199224675,5,312.42233685099995,0.026329204184308102,39.529824033964466,deepset/bert-large-uncased-whole-word-masking-squad2, 3,0.7900724759817968,0.8329492827667042,0.976908815101972,5,305.62878707199707,0.02575668187021718,40.40849724371908,deepset/bert-large-uncased-whole-word-masking-squad2,
4,0.8099612337771785,0.8526275190954586,0.9772459126917242,5,314.3179854819998,0.026488958830439897,39.29142006004379,deepset/xlm-roberta-large-squad2, 4,0.803472105174448,0.846217441464372,0.9742120343839542,5,305.06433064700104,0.025709112645120602,40.48326454229272,deepset/xlm-roberta-large-squad2,
5,0.3730827574582842,0.42342513261953935,0.9539019046013821,5,76.98679084099422,0.006488015408814615,160.417129550279,distilbert-base-uncased-distilled-squad,

1 EM f1 top_n_accuracy top_n reader_time seconds_per_query passages_per_second reader error
2 0 0.7589752233271532 0.7836676217765043 0.8067985794671885 0.8262983412843887 0.9671329849991572 0.9742963087813922 5 133.79706027999998 124.91606550999859 0.011275666634080564 0.01052722615118815 92.30397120949361 98.86638639776463 deepset/roberta-base-squad2
3 1 0.7359683128265633 0.7439743805831789 0.7823306265318686 0.7890026641413856 0.9714309792684982 0.9720209000505646 5 125.22323393199997 67.87064415001078 0.010553112584864317 0.005719757639475036 98.62387044489225 181.96379531485616 deepset/minilm-uncased-squad2
4 2 0.700825889094893 0.6947581324793528 0.7490271600053505 0.7432668866064459 0.9585369964604753 0.9557559413450194 5 123.58959278499992 116.45726653200109 0.010415438461570867 0.009814365964267747 99.92750782409666 106.04748306200683 deepset/bert-base-cased-squad2
5 3 0.7821506826226192 0.7900724759817968 0.8264545708097472 0.8329492827667042 0.9762346199224675 0.976908815101972 5 312.42233685099995 305.62878707199707 0.026329204184308102 0.02575668187021718 39.529824033964466 40.40849724371908 deepset/bert-large-uncased-whole-word-masking-squad2
6 4 0.8099612337771785 0.803472105174448 0.8526275190954586 0.846217441464372 0.9772459126917242 0.9742120343839542 5 314.3179854819998 305.06433064700104 0.026488958830439897 0.025709112645120602 39.29142006004379 40.48326454229272 deepset/xlm-roberta-large-squad2
7 5 0.3730827574582842 0.42342513261953935 0.9539019046013821 5 76.98679084099422 0.006488015408814615 160.417129550279 distilbert-base-uncased-distilled-squad

8
test/benchmarks/retriever.py
View File

@ -167,6 +167,10 @@ def benchmark_querying(n_docs_options,
"error": None "error": None
} }
logger.info("Deleting all docs from this run ...")
if isinstance(doc_store, FAISSDocumentStore):
doc_store.session.close()
else:
doc_store.delete_all_documents(index=doc_index) doc_store.delete_all_documents(index=doc_index)
doc_store.delete_all_documents(index=label_index) doc_store.delete_all_documents(index=label_index)
time.sleep(5) time.sleep(5)
@ -190,6 +194,10 @@ def benchmark_querying(n_docs_options,
"date_time": datetime.datetime.now(), "date_time": datetime.datetime.now(),
"error": str(tb) "error": str(tb)
} }
logger.info("Deleting all docs from this run ...")
if isinstance(doc_store, FAISSDocumentStore):
doc_store.session.close()
else:
doc_store.delete_all_documents(index=doc_index) doc_store.delete_all_documents(index=doc_index)
doc_store.delete_all_documents(index=label_index) doc_store.delete_all_documents(index=label_index)
time.sleep(5) time.sleep(5)

26
test/benchmarks/retriever_index_results.csv
View File

@ -1,13 +1,13 @@
retriever,doc_store,n_docs,indexing_time,docs_per_second,date_time,Notes ,retriever,doc_store,n_docs,indexing_time,docs_per_second,date_time,error
dpr,elasticsearch,1000,14.16526405,70.59522482,2020-10-08 10:30:56, 1,dpr,elasticsearch,10000,135.8048727600035,73.63506033890373,2020-12-02 06:51:48.587178,
elastic,elasticsearch,1000,5.805040058,172.2640998,2020-10-08 10:30:25, 5,dpr,elasticsearch,100000,1352.514667440999,73.93635160290218,2020-12-02 07:23:04.264694,
elastic,elasticsearch,10000,22.56448254,443.1743553,2020-10-08 13:01:09, 9,dpr,elasticsearch,500000,6781.024389943996,73.7351720400064,2020-12-02 10:10:42.147031,
dpr,elasticsearch,10000,126.2442168,79.21154929,2020-10-08 13:03:32, 0,elastic,elasticsearch,10000,20.694342684997537,483.223852635317,2020-12-02 06:49:00.317977,
dpr,elasticsearch,100000,1257.202958,79.54165185,2020-10-08 13:28:16, 4,elastic,elasticsearch,100000,206.47108666299755,484.32931514144724,2020-12-02 06:59:54.055199,
elastic,elasticsearch,100000,209.681252,476.9143596,2020-10-08 13:07:05, 8,elastic,elasticsearch,500000,1032.1480222880054,484.4266415311529,2020-12-02 08:16:15.828533,
dpr,faiss_flat,1000,8.223732258,121.5992895,44112.24392, 2,dpr,faiss_flat,10000,95.10171413100034,105.15057579535569,2020-12-02 06:53:59.472952,
dpr,faiss_flat,10000,89.72649358,111.4498026,44112.24663, 6,dpr,faiss_flat,100000,954.4610684969957,104.77116699738367,2020-12-02 07:39:56.194345,
dpr,faiss_flat,100000,927.0740565,107.8662479,44112.56656, 10,dpr,faiss_flat,500000,4865.149988802004,102.77175444761984,2020-12-02 11:34:34.726687,
dpr,faiss_hnsw,1000,8.86507699,112.8021788,44113.37262,"hnsw 128,20,80" 3,dpr,faiss_hnsw,10000,103.25490099400486,96.84770314757859,2020-12-02 06:56:14.230579,
dpr,faiss_hnsw,10000,100.1804832,99.81984193,44113.37413,"hnsw 128,20,80" 7,dpr,faiss_hnsw,100000,1093.9618158599915,91.41086878008392,2020-12-02 07:58:43.508489,
dpr,faiss_hnsw,100000,1084.063917,92.24548333,44113.38721,"hnsw 128,20,80" 11,dpr,faiss_hnsw,500000,5784.850161597002,86.43266221816312,2020-12-02 13:11:43.328380,

1 retriever doc_store n_docs indexing_time docs_per_second date_time Notes error
2 1 dpr elasticsearch 1000 10000 14.16526405 135.8048727600035 70.59522482 73.63506033890373 2020-10-08 10:30:56 2020-12-02 06:51:48.587178
3 5 elastic dpr elasticsearch 1000 100000 5.805040058 1352.514667440999 172.2640998 73.93635160290218 2020-10-08 10:30:25 2020-12-02 07:23:04.264694
4 9 elastic dpr elasticsearch 10000 500000 22.56448254 6781.024389943996 443.1743553 73.7351720400064 2020-10-08 13:01:09 2020-12-02 10:10:42.147031
5 0 dpr elastic elasticsearch 10000 126.2442168 20.694342684997537 79.21154929 483.223852635317 2020-10-08 13:03:32 2020-12-02 06:49:00.317977
6 4 dpr elastic elasticsearch 100000 1257.202958 206.47108666299755 79.54165185 484.32931514144724 2020-10-08 13:28:16 2020-12-02 06:59:54.055199
7 8 elastic elasticsearch 100000 500000 209.681252 1032.1480222880054 476.9143596 484.4266415311529 2020-10-08 13:07:05 2020-12-02 08:16:15.828533
8 2 dpr faiss_flat 1000 10000 8.223732258 95.10171413100034 121.5992895 105.15057579535569 44112.24392 2020-12-02 06:53:59.472952
9 6 dpr faiss_flat 10000 100000 89.72649358 954.4610684969957 111.4498026 104.77116699738367 44112.24663 2020-12-02 07:39:56.194345
10 10 dpr faiss_flat 100000 500000 927.0740565 4865.149988802004 107.8662479 102.77175444761984 44112.56656 2020-12-02 11:34:34.726687
11 3 dpr faiss_hnsw 1000 10000 8.86507699 103.25490099400486 112.8021788 96.84770314757859 44113.37262 2020-12-02 06:56:14.230579 hnsw 128,20,80
12 7 dpr faiss_hnsw 10000 100000 100.1804832 1093.9618158599915 99.81984193 91.41086878008392 44113.37413 2020-12-02 07:58:43.508489 hnsw 128,20,80
13 11 dpr faiss_hnsw 100000 500000 1084.063917 5784.850161597002 92.24548333 86.43266221816312 44113.38721 2020-12-02 13:11:43.328380 hnsw 128,20,80

30
test/benchmarks/retriever_query_results.csv
View File

@ -1,17 +1,13 @@
retriever,doc_store,n_docs,n_queries,retrieve_time,queries_per_second,seconds_per_query,recall,map,top_k,date_time,error,name,note ,retriever,doc_store,n_docs,n_queries,retrieve_time,queries_per_second,seconds_per_query,recall,map,top_k,date_time,error
dpr,elasticsearch,1000,1085,26.592,40.802,0.025,0.991,0.929,10,2020-10-07 15:06:57,,dpr-elasticsearch, 1,dpr,elasticsearch,10000,5791,233.54168710688828,24.796429587106445,0.040328386652890395,0.9690899671904679,0.8808447974826822,10,2020-12-02 13:18:27.808539,
dpr,elasticsearch,10000,5791,214.425,27.007,0.037,0.975,0.898,10,2020-10-07 15:11:35,,dpr-elasticsearch, 5,dpr,elasticsearch,100000,5791,928.9148432369257,6.234155953220104,0.1604066384453334,0.9397340701087895,0.8212235461156204,10,2020-12-02 13:53:44.689757,
dpr,elasticsearch,100000,5791,886.045,6.536,0.153,0.958,0.863,10,2020-10-07 15:30:52,,dpr-elasticsearch, 9,dpr,elasticsearch,500000,5791,3992.798643678747,1.45036114184423,0.6894834473629333,0.8919012260404076,0.7302081363253893,10,2020-12-02 17:35:25.795083,
dpr,elasticsearch,500000,5791,3824.624,1.514,0.660,0.930,0.805,10,2020-10-07 17:44:02,,dpr-elasticsearch, 0,elastic,elasticsearch,10000,5791,23.260322959773475,248.9647289083211,0.00401663321702184,0.8103954412018649,0.6609973604361457,10,2020-12-02 13:13:03.957613,
dpr,faiss_flat,1000,1085,27.092,40.048,0.025,0.991,0.929,10,2020-10-07 13:06:35,,dpr-faiss_flat, 4,elastic,elasticsearch,100000,5791,35.61682877641579,162.59167924109505,0.006150376234918976,0.7168019340355725,0.559593430418849,10,2020-12-02 13:33:30.417021,
dpr,faiss_flat,10000,5791,241.524,23.977,0.042,0.975,0.898,10,2020-10-07 13:17:21,,dpr-faiss_flat, 8,elastic,elasticsearch,500000,5791,63.36918604133825,91.38510941614904,0.010942701785760362,0.6238991538594371,0.45245893326535686,10,2020-12-02 16:08:13.070376,
dpr,faiss_flat,100000,5791,1148.181,5.044,0.198,0.958,0.863,10,2020-10-07 14:04:51,,dpr-faiss_flat, 2,dpr,faiss_flat,10000,5791,257.67369354520633,22.474160712040344,0.044495543696288435,0.9746157831117251,0.8978985590667505,10,2020-12-02 13:23:51.002905,
dpr,faiss_flat,500000,5791,5308.016,1.091,0.917,0.930,0.805,10,2020-10-08 10:01:32,,dpr-faiss_flat, 6,dpr,faiss_flat,100000,5791,1182.7107160334417,4.896379073508164,0.2042325532780939,0.9575202901053359,0.8630120493486063,10,2020-12-02 14:18:14.837806,
elastic,elasticsearch,1000,1085,4.657,232.978,0.004,0.891,0.748,10,2020-10-07 13:04:47,,elastic-elasticsearch, 3,dpr,faiss_hnsw,10000,5791,184.7552210999711,31.34417509568776,0.03190385444655001,0.972198238646175,0.8961883245210815,10,2020-12-02 13:28:33.415220,
elastic,elasticsearch,10000,5791,34.509,167.810,0.006,0.811,0.661,10,2020-10-07 13:07:52,,elastic-elasticsearch, 7,dpr,faiss_hnsw,100000,5791,450.7693457186833,12.84692505158515,0.0778396383558424,0.9399067518563288,0.8486882354392283,10,2020-12-02 15:10:44.114148,
elastic,elasticsearch,100000,5791,35.529,162.996,0.006,0.717,0.560,10,2020-10-07 13:21:48,,elastic-elasticsearch, 8,dpr,faiss_flat,500000,5791,5365.806154628852,1.0792413727067556,0.9265767837383616,0.9295458470039717,0.8045832613826054,10,2020-12-02 23:14:44.503864,
elastic,elasticsearch,500000,5791,60.645,95.491,0.010,0.624,0.452,10,2020-10-07 16:14:52,,elastic-elasticsearch, 9,dpr,faiss_hnsw,500000,5791,1745.922715222303,3.3168707580865915,0.30148898553312087,0.8820583664306683,0.765677378416975,10,2020-12-03 00:18:53.376265,
dpr,faiss_hnsw,1000,1085,28.640,37.884,0.026,0.991,0.929,10,2020-10-09 07:19:29,,dpr-faiss_hnsw,"128,20,80"
dpr,faiss_hnsw,10000,5791,173.272,33.421,0.030,0.972,0.896,10,2020-10-09 07:23:28,,dpr-faiss_hnsw,"128,20,80"
dpr,faiss_hnsw,100000,5791,451.884,12.815,0.078,0.940,0.849,10,2020-10-09 07:37:56,,dpr-faiss_hnsw,"128,20,80"
dpr,faiss_hnsw,500000,5791,1777.023,3.259,0.307,0.882,0.766,10,2020-10-09,,dpr-faiss_hnsw,"128,20,80"

1 retriever doc_store n_docs n_queries retrieve_time queries_per_second seconds_per_query recall map top_k date_time name error note
2 1 dpr elasticsearch 1000 10000 1085 5791 26.592 233.54168710688828 40.802 24.796429587106445 0.025 0.040328386652890395 0.991 0.9690899671904679 0.929 0.8808447974826822 10 2020-10-07 15:06:57 2020-12-02 13:18:27.808539 dpr-elasticsearch
3 5 dpr elasticsearch 10000 100000 5791 214.425 928.9148432369257 27.007 6.234155953220104 0.037 0.1604066384453334 0.975 0.9397340701087895 0.898 0.8212235461156204 10 2020-10-07 15:11:35 2020-12-02 13:53:44.689757 dpr-elasticsearch
4 9 dpr elasticsearch 100000 500000 5791 886.045 3992.798643678747 6.536 1.45036114184423 0.153 0.6894834473629333 0.958 0.8919012260404076 0.863 0.7302081363253893 10 2020-10-07 15:30:52 2020-12-02 17:35:25.795083 dpr-elasticsearch
5 0 dpr elastic elasticsearch 500000 10000 5791 3824.624 23.260322959773475 1.514 248.9647289083211 0.660 0.00401663321702184 0.930 0.8103954412018649 0.805 0.6609973604361457 10 2020-10-07 17:44:02 2020-12-02 13:13:03.957613 dpr-elasticsearch
6 4 dpr elastic faiss_flat elasticsearch 1000 100000 1085 5791 27.092 35.61682877641579 40.048 162.59167924109505 0.025 0.006150376234918976 0.991 0.7168019340355725 0.929 0.559593430418849 10 2020-10-07 13:06:35 2020-12-02 13:33:30.417021 dpr-faiss_flat
7 8 dpr elastic faiss_flat elasticsearch 10000 500000 5791 241.524 63.36918604133825 23.977 91.38510941614904 0.042 0.010942701785760362 0.975 0.6238991538594371 0.898 0.45245893326535686 10 2020-10-07 13:17:21 2020-12-02 16:08:13.070376 dpr-faiss_flat
8 2 dpr faiss_flat 100000 10000 5791 1148.181 257.67369354520633 5.044 22.474160712040344 0.198 0.044495543696288435 0.958 0.9746157831117251 0.863 0.8978985590667505 10 2020-10-07 14:04:51 2020-12-02 13:23:51.002905 dpr-faiss_flat
9 6 dpr faiss_flat 500000 100000 5791 5308.016 1182.7107160334417 1.091 4.896379073508164 0.917 0.2042325532780939 0.930 0.9575202901053359 0.805 0.8630120493486063 10 2020-10-08 10:01:32 2020-12-02 14:18:14.837806 dpr-faiss_flat
10 3 elastic dpr elasticsearch faiss_hnsw 1000 10000 1085 5791 4.657 184.7552210999711 232.978 31.34417509568776 0.004 0.03190385444655001 0.891 0.972198238646175 0.748 0.8961883245210815 10 2020-10-07 13:04:47 2020-12-02 13:28:33.415220 elastic-elasticsearch
11 7 elastic dpr elasticsearch faiss_hnsw 10000 100000 5791 34.509 450.7693457186833 167.810 12.84692505158515 0.006 0.0778396383558424 0.811 0.9399067518563288 0.661 0.8486882354392283 10 2020-10-07 13:07:52 2020-12-02 15:10:44.114148 elastic-elasticsearch
12 8 elastic dpr elasticsearch faiss_flat 100000 500000 5791 35.529 5365.806154628852 162.996 1.0792413727067556 0.006 0.9265767837383616 0.717 0.9295458470039717 0.560 0.8045832613826054 10 2020-10-07 13:21:48 2020-12-02 23:14:44.503864 elastic-elasticsearch
13 9 elastic dpr elasticsearch faiss_hnsw 500000 5791 60.645 1745.922715222303 95.491 3.3168707580865915 0.010 0.30148898553312087 0.624 0.8820583664306683 0.452 0.765677378416975 10 2020-10-07 16:14:52 2020-12-03 00:18:53.376265 elastic-elasticsearch
dpr faiss_hnsw 1000 1085 28.640 37.884 0.026 0.991 0.929 10 2020-10-09 07:19:29 dpr-faiss_hnsw 128,20,80
dpr faiss_hnsw 10000 5791 173.272 33.421 0.030 0.972 0.896 10 2020-10-09 07:23:28 dpr-faiss_hnsw 128,20,80
dpr faiss_hnsw 100000 5791 451.884 12.815 0.078 0.940 0.849 10 2020-10-09 07:37:56 dpr-faiss_hnsw 128,20,80
dpr faiss_hnsw 500000 5791 1777.023 3.259 0.307 0.882 0.766 10 2020-10-09 dpr-faiss_hnsw 128,20,80

4
test/benchmarks/templates.py
View File

@ -25,12 +25,12 @@ RETRIEVER_TEMPLATE = {
"Query Speed (queries/sec)" "Query Speed (queries/sec)"
], ],
"series": { "series": {
"s0": "recall", "s0": "map",
"s1": "time", "s1": "time",
"s2": "time" "s2": "time"
}, },
"axes": { "axes": {
"label": "recall", "label": "map",
"time_side": "top", "time_side": "top",
"time_label": "seconds" "time_label": "seconds"
}, },