mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-09-26 16:46:58 +00:00
refactor: Delete outdated benchmark files (#5008)
This commit is contained in:
bogdankostic
GitHub
parent
8487cddc69
commit
c3e59914da
@ -1,74 +0,0 @@
|
||||
{
|
||||
"params": {
|
||||
"full": {
|
||||
"retriever_doc_stores": [
|
||||
[
|
||||
"elastic",
|
||||
"elasticsearch"
|
||||
],
|
||||
[
|
||||
"elastic",
|
||||
"opensearch_flat"
|
||||
],
|
||||
[
|
||||
"dpr",
|
||||
"opensearch_flat"
|
||||
],
|
||||
[
|
||||
"dpr",
|
||||
"opensearch_hnsw"
|
||||
],
|
||||
[
|
||||
"dpr",
|
||||
"elasticsearch"
|
||||
],
|
||||
[
|
||||
"dpr",
|
||||
"faiss_flat"
|
||||
],
|
||||
[
|
||||
"dpr",
|
||||
"faiss_hnsw"
|
||||
],
|
||||
[
|
||||
"sentence_transformers",
|
||||
"elasticsearch"
|
||||
]
|
||||
],
|
||||
"n_docs_options": [
|
||||
1000,
|
||||
10000,
|
||||
100000,
|
||||
500000
|
||||
],
|
||||
"n_queries": null
|
||||
},
|
||||
"ci": {
|
||||
"retriever_doc_stores": [
|
||||
[
|
||||
"elastic",
|
||||
"elasticsearch"
|
||||
]
|
||||
],
|
||||
"n_docs_options": [
|
||||
1000
|
||||
],
|
||||
"n_queries": 100
|
||||
}
|
||||
},
|
||||
"filenames": {
|
||||
"data_s3_url": "https://ext-haystack-retriever-eval.s3-eu-west-1.amazonaws.com/",
|
||||
"data_dir": "../../data/retriever/",
|
||||
"filename_gold": "nq2squad-dev.json",
|
||||
"filenames_negative": {
|
||||
"10000": "psgs_w100_minus_gold_10k.tsv",
|
||||
"100000": "psgs_w100_minus_gold_100k.tsv",
|
||||
"1000000": "psgs_w100_minus_gold_1m.tsv"
|
||||
},
|
||||
"embeddings_dir": "embeddings/",
|
||||
"embeddings_filenames": {
|
||||
"10000": "wikipedia_passages_10k.pkl",
|
||||
"100000": "wikipedia_passages_100k.pkl",
|
||||
"1000000": "wikipedia_passages_1m.pkl"}
|
||||
}
|
||||
}
|
||||
@ -1,49 +0,0 @@
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
from tqdm import tqdm
|
||||
import json
|
||||
|
||||
n_passages = 1_000_000
|
||||
embeddings_dir = Path("embeddings")
|
||||
embeddings_filenames = [f"wikipedia_passages_{i}.pkl" for i in range(50)]
|
||||
neg_passages_filename = "psgs_w100_minus_gold.tsv"
|
||||
gold_passages_filename = "nq2squad-dev.json"
|
||||
|
||||
# Extract gold passage ids
|
||||
passage_ids = []
|
||||
gold_data = json.load(open(gold_passages_filename))["data"]
|
||||
for d in gold_data:
|
||||
for p in d["paragraphs"]:
|
||||
passage_ids.append(str(p["passage_id"]))
|
||||
print("gold_ids")
|
||||
print(len(passage_ids))
|
||||
print()
|
||||
|
||||
# Extract neg passage ids
|
||||
with open(neg_passages_filename) as f:
|
||||
f.readline() # Ignore column headers
|
||||
for _ in range(n_passages - len(passage_ids)):
|
||||
l = f.readline()
|
||||
passage_ids.append(str(l.split()[0]))
|
||||
assert len(passage_ids) == len(set(passage_ids))
|
||||
assert set([type(x) for x in passage_ids]) == {str}
|
||||
passage_ids = set(passage_ids)
|
||||
print("all_ids")
|
||||
print(len(passage_ids))
|
||||
print()
|
||||
|
||||
|
||||
# Gather vectors for passages
|
||||
ret = []
|
||||
for ef in tqdm(embeddings_filenames):
|
||||
curr = pickle.load(open(embeddings_dir / ef, "rb"))
|
||||
for i, vec in curr:
|
||||
if i in passage_ids:
|
||||
ret.append((i, vec))
|
||||
print("n_vectors")
|
||||
print(len(ret))
|
||||
print()
|
||||
|
||||
# Write vectors to file
|
||||
with open(f"wikipedia_passages_{n_passages}.pkl", "wb") as f:
|
||||
pickle.dump(ret, f)
|
||||
@ -1,21 +0,0 @@
|
||||
import json
|
||||
from tqdm import tqdm
|
||||
import time
|
||||
import random
|
||||
|
||||
random.seed(42)
|
||||
|
||||
lines = []
|
||||
with open("psgs_w100_minus_gold_unshuffled.tsv") as f:
|
||||
f.readline() # Remove column header
|
||||
lines = [l for l in tqdm(f)]
|
||||
|
||||
tic = time.perf_counter()
|
||||
random.shuffle(lines)
|
||||
toc = time.perf_counter()
|
||||
t = toc - tic
|
||||
print(t)
|
||||
with open("psgs_w100_minus_gold.tsv", "w") as f:
|
||||
f.write("id\ttext\title\n")
|
||||
for l in tqdm(lines):
|
||||
f.write(l)
|
||||
@ -1,7 +0,0 @@
|
||||
,EM,f1,top_n_accuracy,top_n,reader_time,seconds_per_query,EM_text_answer,f1_text_answer,top_n_accuracy_text_answer,top_n_EM_text_answer,top_n_f1_text_answer,Total_text_answer,EM_no_answer,f1_no_answer,top_n_accuracy_no_answer,Total_no_answer,passages_per_second,reader,error
|
||||
0,76.1672004045171,78.21542223321491,97.41277599865161,5,104.290868674987,0.0087890501158762,61.491731353358084,65.59301387433817,94.81943975700304,81.01586230172123,90.50667071721769,5926,90.8080808080808,90.8080808080808,100.0,5940,118.41880460779026,deepset/roberta-base-squad2,
|
||||
1,76.84982302376538,79.6479775678566,97.21894488454407,5,52.54729196999688,0.004428391367773207,65.89605129935876,71.49897094501964,94.4313196085049,79.96962537968275,89.70802565150441,5926,87.77777777777777,87.77777777777777,100.0,5940,235.02638360605766,deepset/minilm-uncased-squad2,
|
||||
2,70.0825889094893,73.49806294434092,95.53345693578291,5,105.55706145399017,0.008895757749367114,56.59804252446844,63.43705955071704,91.05636179547756,72.1228484643942,84.47853540832811,5926,83.53535353535354,83.53535353535354,100.0,5940,116.99833085428466,deepset/bert-base-cased-squad2,
|
||||
3,83.69290409573573,87.35038229973254,98.07854373841226,5,284.2086430560157,0.023951512140233923,75.19406007424907,82.51765716649112,96.15254809314884,82.33209584880188,91.89205932239825,5926,92.17171717171718,92.17171717171718,100.0,5940,43.45399164221017,deepset/roberta-large-squad2,
|
||||
4,79.50446654306421,82.29388851439239,97.52233271532108,5,103.88512606802396,0.008754856402159445,71.110361120486,76.69579499017553,95.03881201484981,81.72460344245697,90.89098996208568,5926,87.87878787878788,87.87878787878788,100.0,5940,118.88131118898794,deepset/roberta-base-squad2-distilled,
|
||||
5,38.53868194842407,43.340283031558755,95.43232765885725,5,62.23228176299017,0.005244588046771462,74.53594330070874,84.1504891077415,90.85386432669593,75.09281133985826,85.21259442024737,5926,2.6262626262626263,2.6262626262626263,100.0,5940,198.4500591997352,distilbert-base-uncased-distilled-squad,
|
||||
|
@ -1,8 +0,0 @@
|
||||
| | EM | f1 | top_n_accuracy | top_n | reader_time | seconds_per_query | EM_text_answer | f1_text_answer | top_n_accuracy_text_answer | top_n_EM_text_answer | top_n_f1_text_answer | Total_text_answer | EM_no_answer | f1_no_answer | top_n_accuracy_no_answer | Total_no_answer | passages_per_second | reader | error |
|
||||
|---:|--------:|--------:|-----------------:|--------:|--------------:|--------------------:|-----------------:|-----------------:|-----------------------------:|-----------------------:|-----------------------:|--------------------:|---------------:|---------------:|---------------------------:|------------------:|----------------------:|:----------------------------------------|:--------|
|
||||
| 0 | 76.1672 | 78.2154 | 97.4128 | 5 | 104.291 | 0.00878905 | 61.4917 | 65.593 | 94.8194 | 81.0159 | 90.5067 | 5926 | 90.8081 | 90.8081 | 100 | 5940 | 118.419 | deepset/roberta-base-squad2 | |
|
||||
| 1 | 76.8498 | 79.648 | 97.2189 | 5 | 52.5473 | 0.00442839 | 65.8961 | 71.499 | 94.4313 | 79.9696 | 89.708 | 5926 | 87.7778 | 87.7778 | 100 | 5940 | 235.026 | deepset/minilm-uncased-squad2 | |
|
||||
| 2 | 70.0826 | 73.4981 | 95.5335 | 5 | 105.557 | 0.00889576 | 56.598 | 63.4371 | 91.0564 | 72.1228 | 84.4785 | 5926 | 83.5354 | 83.5354 | 100 | 5940 | 116.998 | deepset/bert-base-cased-squad2 | |
|
||||
| 3 | 83.6929 | 87.3504 | 98.0785 | 5 | 284.209 | 0.0239515 | 75.1941 | 82.5177 | 96.1525 | 82.3321 | 91.8921 | 5926 | 92.1717 | 92.1717 | 100 | 5940 | 43.454 | deepset/roberta-large-squad2 | |
|
||||
| 4 | 79.5045 | 82.2939 | 97.5223 | 5 | 103.885 | 0.00875486 | 71.1104 | 76.6958 | 95.0388 | 81.7246 | 90.891 | 5926 | 87.8788 | 87.8788 | 100 | 5940 | 118.881 | deepset/roberta-base-squad2-distilled | |
|
||||
| 5 | 38.5387 | 43.3403 | 95.4323 | 5 | 62.2323 | 0.00524459 | 74.5359 | 84.1505 | 90.8539 | 75.0928 | 85.2126 | 5926 | 2.62626 | 2.62626 | 100 | 5940 | 198.45 | distilbert-base-uncased-distilled-squad | |
|
||||
@ -1,106 +0,0 @@
|
||||
import json
|
||||
import pandas as pd
|
||||
from pprint import pprint
|
||||
|
||||
|
||||
def reader(reader_csv="reader_results.csv"):
|
||||
model_rename_map = {
|
||||
"deepset/roberta-base-squad2": "RoBERTa",
|
||||
"deepset/minilm-uncased-squad2": "MiniLM",
|
||||
"deepset/bert-base-cased-squad2": "BERT base",
|
||||
"deepset/bert-large-uncased-whole-word-masking-squad2": "BERT large",
|
||||
"deepset/xlm-roberta-large-squad2": "XLM-RoBERTa",
|
||||
"deepset/roberta-large-squad2": "RoBERTa large",
|
||||
"deepset/roberta-base-squad2-distilled": "RoBERTa distilled",
|
||||
"distilbert-base-uncased-distilled-squad": "distilBERT",
|
||||
}
|
||||
|
||||
column_name_map = {"f1": "F1", "passages_per_second": "Speed", "reader": "Model"}
|
||||
|
||||
df = pd.read_csv(reader_csv)
|
||||
df = df[["f1", "passages_per_second", "reader"]]
|
||||
df["reader"] = df["reader"].map(model_rename_map)
|
||||
df = df[list(column_name_map)]
|
||||
df = df.rename(columns=column_name_map)
|
||||
ret = [dict(row) for i, row in df.iterrows()]
|
||||
print("Reader overview")
|
||||
print(json.dumps(ret, indent=4))
|
||||
return ret
|
||||
|
||||
|
||||
def retriever(index_csv="retriever_index_results.csv", query_csv="retriever_query_results.csv"):
|
||||
column_name_map = {
|
||||
"model": "model",
|
||||
"n_docs": "n_docs",
|
||||
"docs_per_second": "index_speed",
|
||||
"queries_per_second": "query_speed",
|
||||
"map": "map",
|
||||
}
|
||||
|
||||
name_cleaning = {
|
||||
"dpr": "DPR",
|
||||
"elastic": "BM25",
|
||||
"elasticsearch": "Elasticsearch",
|
||||
"faiss": "FAISS",
|
||||
"faiss_flat": "FAISS (flat)",
|
||||
"faiss_hnsw": "FAISS (HNSW)",
|
||||
"sentence_transformers": "Sentence Transformers",
|
||||
"opensearch_flat": "OpenSearch (flat)",
|
||||
"opensearch_hnsw": "OpenSearch (HNSW)",
|
||||
}
|
||||
|
||||
index = pd.read_csv(index_csv)
|
||||
query = pd.read_csv(query_csv)
|
||||
df = pd.merge(
|
||||
index,
|
||||
query,
|
||||
how="right",
|
||||
left_on=["retriever", "doc_store", "n_docs"],
|
||||
right_on=["retriever", "doc_store", "n_docs"],
|
||||
)
|
||||
|
||||
df["retriever"] = df["retriever"].map(name_cleaning)
|
||||
df["doc_store"] = df["doc_store"].map(name_cleaning)
|
||||
df["model"] = df["retriever"] + " / " + df["doc_store"]
|
||||
|
||||
df = df[list(column_name_map)]
|
||||
df = df.rename(columns=column_name_map)
|
||||
|
||||
print("Retriever overview")
|
||||
retriever_overview_data = retriever_overview(df)
|
||||
print(json.dumps(retriever_overview_data, indent=4))
|
||||
|
||||
print("Retriever MAP")
|
||||
retriever_map_data = retriever_map(df)
|
||||
print(json.dumps(retriever_map_data, indent=4))
|
||||
|
||||
print("Retriever Speed")
|
||||
retriever_speed_data = retriever_speed(df)
|
||||
print(json.dumps(retriever_speed_data, indent=4))
|
||||
|
||||
return retriever_overview_data, retriever_map_data, retriever_speed_data
|
||||
|
||||
|
||||
def retriever_map(df):
|
||||
columns = ["model", "n_docs", "map"]
|
||||
df = df[columns]
|
||||
ret = df.to_dict(orient="records")
|
||||
return ret
|
||||
|
||||
|
||||
def retriever_speed(df):
|
||||
columns = ["model", "n_docs", "query_speed"]
|
||||
df = df[columns]
|
||||
ret = df.to_dict(orient="records")
|
||||
return ret
|
||||
|
||||
|
||||
def retriever_overview(df, chosen_n_docs=100_000):
|
||||
df = df[df["n_docs"] == chosen_n_docs]
|
||||
ret = [dict(row) for i, row in df.iterrows()]
|
||||
return ret
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
reader()
|
||||
retriever()
|
||||
@ -1,33 +0,0 @@
|
||||
,retriever,doc_store,n_docs,indexing_time,docs_per_second,date_time,error
|
||||
31,sentence_transformers,elasticsearch,500000,4165.329931593005,120.03851032486591,2022-09-17 21:16:26.676611,
|
||||
28,dpr,elasticsearch,500000,6180.3500814459985,80.90156599721556,2022-09-17 16:45:36.329915,
|
||||
7,sentence_transformers,elasticsearch,1000,9.844315000998904,101.5814711230319,2022-09-17 08:26:44.750140,
|
||||
15,sentence_transformers,elasticsearch,10000,84.88162445399212,117.8111289024663,2022-09-17 08:43:51.615496,
|
||||
20,dpr,elasticsearch,100000,1230.761674939,81.2504988059173,2022-09-17 09:57:09.456572,
|
||||
0,elastic,elasticsearch,1000,1.7988810280075995,555.9011321096525,2022-09-17 08:21:11.001120,
|
||||
24,elastic,elasticsearch,500000,1031.7205082759901,484.6273733915616,2022-09-17 11:09:14.536450,
|
||||
16,elastic,elasticsearch,100000,206.83811076599522,483.4698964792531,2022-09-17 08:47:59.723168,
|
||||
8,elastic,elasticsearch,10000,20.81922616600059,480.3252493760203,2022-09-17 08:27:43.460016,
|
||||
4,dpr,elasticsearch,1000,12.114610354998149,82.54495775734364,2022-09-17 08:24:34.582262,
|
||||
12,dpr,elasticsearch,10000,123.21874477900565,81.15648327642944,2022-09-17 08:37:02.872786,
|
||||
23,sentence_transformers,elasticsearch,100000,831.661083732004,120.24128813537725,2022-09-17 10:50:54.888143,
|
||||
29,dpr,faiss_flat,500000,5485.974153257004,91.1415158059306,2022-09-17 18:19:50.024476,
|
||||
13,dpr,faiss_flat,10000,105.0845138200093,95.16150036272884,2022-09-17 08:39:25.198922,
|
||||
5,dpr,faiss_flat,1000,10.151226033005514,98.51026829159531,2022-09-17 08:25:18.819509,
|
||||
21,dpr,faiss_flat,100000,1068.483952116003,93.59054930302146,2022-09-17 10:15:58.178598,
|
||||
22,dpr,faiss_hnsw,100000,1195.8989856069966,83.61910261947709,2022-09-17 10:36:24.783587,
|
||||
14,dpr,faiss_hnsw,10000,114.6786555860017,87.20018515128682,2022-09-17 08:41:49.541725,
|
||||
6,dpr,faiss_hnsw,1000,10.710903742990922,93.36280336329109,2022-09-17 08:25:58.505307,
|
||||
30,dpr,faiss_hnsw,500000,6349.903588575005,78.7413530025274,2022-09-17 20:06:17.374842,
|
||||
9,elastic,opensearch_flat,10000,21.9185139649926,456.2353093814486,2022-09-17 08:28:40.051299,
|
||||
10,dpr,opensearch_flat,10000,123.36043137000524,81.06327036102982,2022-09-17 08:31:26.201876,
|
||||
17,elastic,opensearch_flat,100000,208.89336037798785,478.7131568904452,2022-09-17 08:52:13.178937,
|
||||
1,elastic,opensearch_flat,1000,2.2300678319879808,448.4168533602657,2022-09-17 08:21:45.610495,
|
||||
18,dpr,opensearch_flat,100000,1245.0073385680007,80.32081169498916,2022-09-17 09:13:46.979550,
|
||||
25,elastic,opensearch_flat,500000,1045.8889810429973,478.06221220667453,2022-09-17 11:28:08.906406,
|
||||
26,dpr,opensearch_flat,500000,6235.606822678994,80.18465791997863,2022-09-17 13:13:24.174105,
|
||||
2,dpr,opensearch_flat,1000,12.919973790005315,77.3995378205476,2022-09-17 08:22:43.256934,
|
||||
11,dpr,opensearch_hnsw,10000,121.49061479599914,82.3108848102505,2022-09-17 08:34:11.508717,
|
||||
3,dpr,opensearch_hnsw,1000,12.355953149002744,80.93264744053441,2022-09-17 08:23:36.852546,
|
||||
19,dpr,opensearch_hnsw,100000,1242.960266391994,80.45309468360983,2022-09-17 09:35:31.609351,
|
||||
27,dpr,opensearch_hnsw,500000,6237.192302857991,80.16427516125985,2022-09-17 14:59:57.763007,
|
||||
|
@ -1,34 +0,0 @@
|
||||
| | retriever | doc_store | n_docs | indexing_time | docs_per_second | date_time | error |
|
||||
|---:|:----------------------|:----------------|---------:|----------------:|------------------:|:---------------------------|:--------|
|
||||
| 31 | sentence_transformers | elasticsearch | 500000 | 4165.33 | 120.039 | 2022-09-17 21:16:26.676611 | |
|
||||
| 28 | dpr | elasticsearch | 500000 | 6180.35 | 80.9016 | 2022-09-17 16:45:36.329915 | |
|
||||
| 7 | sentence_transformers | elasticsearch | 1000 | 9.84432 | 101.581 | 2022-09-17 08:26:44.750140 | |
|
||||
| 15 | sentence_transformers | elasticsearch | 10000 | 84.8816 | 117.811 | 2022-09-17 08:43:51.615496 | |
|
||||
| 20 | dpr | elasticsearch | 100000 | 1230.76 | 81.2505 | 2022-09-17 09:57:09.456572 | |
|
||||
| 0 | elastic | elasticsearch | 1000 | 1.79888 | 555.901 | 2022-09-17 08:21:11.001120 | |
|
||||
| 24 | elastic | elasticsearch | 500000 | 1031.72 | 484.627 | 2022-09-17 11:09:14.536450 | |
|
||||
| 16 | elastic | elasticsearch | 100000 | 206.838 | 483.47 | 2022-09-17 08:47:59.723168 | |
|
||||
| 8 | elastic | elasticsearch | 10000 | 20.8192 | 480.325 | 2022-09-17 08:27:43.460016 | |
|
||||
| 4 | dpr | elasticsearch | 1000 | 12.1146 | 82.545 | 2022-09-17 08:24:34.582262 | |
|
||||
| 12 | dpr | elasticsearch | 10000 | 123.219 | 81.1565 | 2022-09-17 08:37:02.872786 | |
|
||||
| 23 | sentence_transformers | elasticsearch | 100000 | 831.661 | 120.241 | 2022-09-17 10:50:54.888143 | |
|
||||
| 29 | dpr | faiss_flat | 500000 | 5485.97 | 91.1415 | 2022-09-17 18:19:50.024476 | |
|
||||
| 13 | dpr | faiss_flat | 10000 | 105.085 | 95.1615 | 2022-09-17 08:39:25.198922 | |
|
||||
| 5 | dpr | faiss_flat | 1000 | 10.1512 | 98.5103 | 2022-09-17 08:25:18.819509 | |
|
||||
| 21 | dpr | faiss_flat | 100000 | 1068.48 | 93.5905 | 2022-09-17 10:15:58.178598 | |
|
||||
| 22 | dpr | faiss_hnsw | 100000 | 1195.9 | 83.6191 | 2022-09-17 10:36:24.783587 | |
|
||||
| 14 | dpr | faiss_hnsw | 10000 | 114.679 | 87.2002 | 2022-09-17 08:41:49.541725 | |
|
||||
| 6 | dpr | faiss_hnsw | 1000 | 10.7109 | 93.3628 | 2022-09-17 08:25:58.505307 | |
|
||||
| 30 | dpr | faiss_hnsw | 500000 | 6349.9 | 78.7414 | 2022-09-17 20:06:17.374842 | |
|
||||
| 9 | elastic | opensearch_flat | 10000 | 21.9185 | 456.235 | 2022-09-17 08:28:40.051299 | |
|
||||
| 10 | dpr | opensearch_flat | 10000 | 123.36 | 81.0633 | 2022-09-17 08:31:26.201876 | |
|
||||
| 17 | elastic | opensearch_flat | 100000 | 208.893 | 478.713 | 2022-09-17 08:52:13.178937 | |
|
||||
| 1 | elastic | opensearch_flat | 1000 | 2.23007 | 448.417 | 2022-09-17 08:21:45.610495 | |
|
||||
| 18 | dpr | opensearch_flat | 100000 | 1245.01 | 80.3208 | 2022-09-17 09:13:46.979550 | |
|
||||
| 25 | elastic | opensearch_flat | 500000 | 1045.89 | 478.062 | 2022-09-17 11:28:08.906406 | |
|
||||
| 26 | dpr | opensearch_flat | 500000 | 6235.61 | 80.1847 | 2022-09-17 13:13:24.174105 | |
|
||||
| 2 | dpr | opensearch_flat | 1000 | 12.92 | 77.3995 | 2022-09-17 08:22:43.256934 | |
|
||||
| 11 | dpr | opensearch_hnsw | 10000 | 121.491 | 82.3109 | 2022-09-17 08:34:11.508717 | |
|
||||
| 3 | dpr | opensearch_hnsw | 1000 | 12.356 | 80.9326 | 2022-09-17 08:23:36.852546 | |
|
||||
| 19 | dpr | opensearch_hnsw | 100000 | 1242.96 | 80.4531 | 2022-09-17 09:35:31.609351 | |
|
||||
| 27 | dpr | opensearch_hnsw | 500000 | 6237.19 | 80.1643 | 2022-09-17 14:59:57.763007 | |
|
||||
@ -1,33 +0,0 @@
|
||||
,retriever,doc_store,n_docs,n_queries,retrieve_time,queries_per_second,seconds_per_query,recall,map,top_k,date_time,error
|
||||
31,sentence_transformers,elasticsearch,500000,5637,3910.198875275557,1.4416146543448518,0.6936666445406345,89.6398793684584,76.49564526892908,10,2022-09-18 08:25:36.627368,
|
||||
28,dpr,elasticsearch,500000,5637,3839.528848723392,1.4681488854743858,0.6811298294701778,93.0814262905801,80.86137228234092,10,2022-09-18 03:20:53.663119,
|
||||
7,sentence_transformers,elasticsearch,1000,1064,22.08566212869482,48.176051675516526,0.020757201248773328,98.7781954887218,90.06638620360428,10,2022-09-17 21:28:29.461918,
|
||||
15,sentence_transformers,elasticsearch,10000,5637,187.41362470001332,30.0778559137467,0.03324705068298977,96.52297321270179,87.11255142468552,10,2022-09-17 22:03:32.200427,
|
||||
20,dpr,elasticsearch,100000,5637,903.5089274104685,6.239008635095737,0.16028187465149343,95.7956359765833,86.5456409043424,10,2022-09-17 22:58:20.213034,
|
||||
0,elastic,elasticsearch,1000,1064,4.122031082282774,258.12517634164914,0.00387408936304772,89.09774436090225,74.2044471297291,10,2022-09-17 21:19:21.768721,
|
||||
24,elastic,elasticsearch,500000,5637,83.56125180030358,67.45949681882962,0.014823709739276845,62.746141564662054,45.603960228760656,10,2022-09-18 00:29:07.528766,
|
||||
16,elastic,elasticsearch,100000,5637,42.696119271276984,132.02605052193084,0.007574262776525986,71.93542664537875,56.26106985872818,10,2022-09-17 22:08:40.215954,
|
||||
8,elastic,elasticsearch,10000,5637,28.68551177883637,196.51035140878582,0.005088790452161854,81.10697179350718,66.32941083712781,10,2022-09-17 21:30:14.745163,
|
||||
4,dpr,elasticsearch,1000,1064,29.50371559121413,36.06325436233689,0.0277290560067802,99.15413533834587,92.9510532283089,10,2022-09-17 21:24:50.768859,
|
||||
12,dpr,elasticsearch,10000,5637,224.79332777825766,25.076367059971158,0.03987818481076063,97.49866950505589,89.87097014904354,10,2022-09-17 21:48:36.651933,
|
||||
23,sentence_transformers,elasticsearch,100000,5637,886.5846678019152,6.358106794217024,0.15727952240587462,93.79102359411034,82.74686664920834,10,2022-09-18 00:09:04.840841,
|
||||
29,dpr,faiss_flat,500000,5637,3683.516663164017,1.5303310709494677,0.6534533729224795,93.0814262905801,80.86137228234091,10,2022-09-18 05:09:24.992364,
|
||||
13,dpr,faiss_flat,10000,5637,181.46477086472441,31.06388073640026,0.032191728022835625,97.49866950505589,89.87097014904354,10,2022-09-17 21:53:56.820064,
|
||||
5,dpr,faiss_flat,1000,1064,29.979966158716707,35.49036694594937,0.028176659923605928,99.15413533834587,92.9510532283089,10,2022-09-17 21:26:11.004926,
|
||||
21,dpr,faiss_flat,100000,5637,841.2282260610373,6.700916380795566,0.14923332021661118,95.7956359765833,86.54606328368972,10,2022-09-17 23:23:47.065373,
|
||||
22,dpr,faiss_hnsw,100000,5637,138.64402735611657,40.6580803190388,0.024595356990618513,93.18786588610963,84.33419639513305,10,2022-09-17 23:39:25.396237,
|
||||
14,dpr,faiss_hnsw,10000,5637,112.71057137736352,50.01305495228924,0.019994779382182637,96.98421145999644,89.51337675393017,10,2022-09-17 21:58:05.961829,
|
||||
6,dpr,faiss_hnsw,1000,1064,25.61499380285386,41.538171283159,0.02407424229591528,99.15413533834587,92.9510532283089,10,2022-09-17 21:27:21.948745,
|
||||
30,dpr,faiss_hnsw,500000,5637,124.99580949536175,45.097511850660666,0.022174172342622273,86.90792974986695,75.73315903145605,10,2022-09-18 06:09:37.007518,
|
||||
9,elastic,opensearch_flat,10000,5637,32.42948475538287,173.8232982275283,0.005752968734323731,81.10697179350718,66.32941083712774,10,2022-09-17 21:32:03.707902,
|
||||
10,dpr,opensearch_flat,10000,5637,182.6267182478914,30.86623936563611,0.03239785670532046,97.49866950505589,89.87097014904354,10,2022-09-17 21:37:08.044354,
|
||||
17,elastic,opensearch_flat,100000,5637,46.926014778233366,120.12526583899732,0.008324643388013725,71.93542664537875,56.26106985872818,10,2022-09-17 22:13:58.492431,
|
||||
1,elastic,opensearch_flat,1000,1064,6.101849522849079,174.3733594241761,0.005734820980121315,89.09774436090225,74.2044471297291,10,2022-09-17 21:20:04.003708,
|
||||
18,dpr,opensearch_flat,100000,5637,320.2046442734136,17.604366772353018,0.056804088038569026,95.76015611140679,86.51945338480733,10,2022-09-17 22:25:11.379134,
|
||||
25,elastic,opensearch_flat,500000,5637,83.39615098570357,67.59304756122785,0.014794420966064143,62.746141564662054,45.59509026246651,10,2022-09-18 00:49:52.903674,
|
||||
26,dpr,opensearch_flat,500000,5637,466.1570336711302,12.092491570076481,0.08269594352867309,93.0814262905801,80.861372282341,10,2022-09-18 01:20:53.923798,
|
||||
2,dpr,opensearch_flat,1000,1064,30.094220374303404,35.35562598951788,0.02828404170517237,99.15413533834587,92.9510532283089,10,2022-09-17 21:21:42.413924,
|
||||
11,dpr,opensearch_hnsw,10000,5637,181.91717168685864,30.986629506879073,0.032271983623710955,97.49866950505589,89.87097014904357,10,2022-09-17 21:42:16.601719,
|
||||
3,dpr,opensearch_hnsw,1000,1064,29.82644817981054,35.67303735213834,0.028032376108844494,99.15413533834587,92.9510532283089,10,2022-09-17 21:23:12.487039,
|
||||
19,dpr,opensearch_hnsw,100000,5637,327.2236303977261,17.226750993345046,0.058049251445401115,95.77789604399504,86.53719331739565,10,2022-09-17 22:36:46.502520,
|
||||
27,dpr,opensearch_hnsw,500000,5637,477.3347775057773,11.809321812785313,0.08467886774982744,93.0814262905801,80.861372282341,10,2022-09-18 01:53:29.735540,
|
||||
|
@ -1,34 +0,0 @@
|
||||
| | retriever | doc_store | n_docs | n_queries | retrieve_time | queries_per_second | seconds_per_query | recall | map | top_k | date_time | error |
|
||||
|---:|:----------------------|:----------------|---------:|------------:|----------------:|---------------------:|--------------------:|---------:|--------:|--------:|:---------------------------|:--------|
|
||||
| 31 | sentence_transformers | elasticsearch | 500000 | 5637 | 3910.2 | 1.44161 | 0.693667 | 89.6399 | 76.4956 | 10 | 2022-09-18 08:25:36.627368 | |
|
||||
| 28 | dpr | elasticsearch | 500000 | 5637 | 3839.53 | 1.46815 | 0.68113 | 93.0814 | 80.8614 | 10 | 2022-09-18 03:20:53.663119 | |
|
||||
| 7 | sentence_transformers | elasticsearch | 1000 | 1064 | 22.0857 | 48.1761 | 0.0207572 | 98.7782 | 90.0664 | 10 | 2022-09-17 21:28:29.461918 | |
|
||||
| 15 | sentence_transformers | elasticsearch | 10000 | 5637 | 187.414 | 30.0779 | 0.0332471 | 96.523 | 87.1126 | 10 | 2022-09-17 22:03:32.200427 | |
|
||||
| 20 | dpr | elasticsearch | 100000 | 5637 | 903.509 | 6.23901 | 0.160282 | 95.7956 | 86.5456 | 10 | 2022-09-17 22:58:20.213034 | |
|
||||
| 0 | elastic | elasticsearch | 1000 | 1064 | 4.12203 | 258.125 | 0.00387409 | 89.0977 | 74.2044 | 10 | 2022-09-17 21:19:21.768721 | |
|
||||
| 24 | elastic | elasticsearch | 500000 | 5637 | 83.5613 | 67.4595 | 0.0148237 | 62.7461 | 45.604 | 10 | 2022-09-18 00:29:07.528766 | |
|
||||
| 16 | elastic | elasticsearch | 100000 | 5637 | 42.6961 | 132.026 | 0.00757426 | 71.9354 | 56.2611 | 10 | 2022-09-17 22:08:40.215954 | |
|
||||
| 8 | elastic | elasticsearch | 10000 | 5637 | 28.6855 | 196.51 | 0.00508879 | 81.107 | 66.3294 | 10 | 2022-09-17 21:30:14.745163 | |
|
||||
| 4 | dpr | elasticsearch | 1000 | 1064 | 29.5037 | 36.0633 | 0.0277291 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:24:50.768859 | |
|
||||
| 12 | dpr | elasticsearch | 10000 | 5637 | 224.793 | 25.0764 | 0.0398782 | 97.4987 | 89.871 | 10 | 2022-09-17 21:48:36.651933 | |
|
||||
| 23 | sentence_transformers | elasticsearch | 100000 | 5637 | 886.585 | 6.35811 | 0.15728 | 93.791 | 82.7469 | 10 | 2022-09-18 00:09:04.840841 | |
|
||||
| 29 | dpr | faiss_flat | 500000 | 5637 | 3683.52 | 1.53033 | 0.653453 | 93.0814 | 80.8614 | 10 | 2022-09-18 05:09:24.992364 | |
|
||||
| 13 | dpr | faiss_flat | 10000 | 5637 | 181.465 | 31.0639 | 0.0321917 | 97.4987 | 89.871 | 10 | 2022-09-17 21:53:56.820064 | |
|
||||
| 5 | dpr | faiss_flat | 1000 | 1064 | 29.98 | 35.4904 | 0.0281767 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:26:11.004926 | |
|
||||
| 21 | dpr | faiss_flat | 100000 | 5637 | 841.228 | 6.70092 | 0.149233 | 95.7956 | 86.5461 | 10 | 2022-09-17 23:23:47.065373 | |
|
||||
| 22 | dpr | faiss_hnsw | 100000 | 5637 | 138.644 | 40.6581 | 0.0245954 | 93.1879 | 84.3342 | 10 | 2022-09-17 23:39:25.396237 | |
|
||||
| 14 | dpr | faiss_hnsw | 10000 | 5637 | 112.711 | 50.0131 | 0.0199948 | 96.9842 | 89.5134 | 10 | 2022-09-17 21:58:05.961829 | |
|
||||
| 6 | dpr | faiss_hnsw | 1000 | 1064 | 25.615 | 41.5382 | 0.0240742 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:27:21.948745 | |
|
||||
| 30 | dpr | faiss_hnsw | 500000 | 5637 | 124.996 | 45.0975 | 0.0221742 | 86.9079 | 75.7332 | 10 | 2022-09-18 06:09:37.007518 | |
|
||||
| 9 | elastic | opensearch_flat | 10000 | 5637 | 32.4295 | 173.823 | 0.00575297 | 81.107 | 66.3294 | 10 | 2022-09-17 21:32:03.707902 | |
|
||||
| 10 | dpr | opensearch_flat | 10000 | 5637 | 182.627 | 30.8662 | 0.0323979 | 97.4987 | 89.871 | 10 | 2022-09-17 21:37:08.044354 | |
|
||||
| 17 | elastic | opensearch_flat | 100000 | 5637 | 46.926 | 120.125 | 0.00832464 | 71.9354 | 56.2611 | 10 | 2022-09-17 22:13:58.492431 | |
|
||||
| 1 | elastic | opensearch_flat | 1000 | 1064 | 6.10185 | 174.373 | 0.00573482 | 89.0977 | 74.2044 | 10 | 2022-09-17 21:20:04.003708 | |
|
||||
| 18 | dpr | opensearch_flat | 100000 | 5637 | 320.205 | 17.6044 | 0.0568041 | 95.7602 | 86.5195 | 10 | 2022-09-17 22:25:11.379134 | |
|
||||
| 25 | elastic | opensearch_flat | 500000 | 5637 | 83.3962 | 67.593 | 0.0147944 | 62.7461 | 45.5951 | 10 | 2022-09-18 00:49:52.903674 | |
|
||||
| 26 | dpr | opensearch_flat | 500000 | 5637 | 466.157 | 12.0925 | 0.0826959 | 93.0814 | 80.8614 | 10 | 2022-09-18 01:20:53.923798 | |
|
||||
| 2 | dpr | opensearch_flat | 1000 | 1064 | 30.0942 | 35.3556 | 0.028284 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:21:42.413924 | |
|
||||
| 11 | dpr | opensearch_hnsw | 10000 | 5637 | 181.917 | 30.9866 | 0.032272 | 97.4987 | 89.871 | 10 | 2022-09-17 21:42:16.601719 | |
|
||||
| 3 | dpr | opensearch_hnsw | 1000 | 1064 | 29.8264 | 35.673 | 0.0280324 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:23:12.487039 | |
|
||||
| 19 | dpr | opensearch_hnsw | 100000 | 5637 | 327.224 | 17.2268 | 0.0580493 | 95.7779 | 86.5372 | 10 | 2022-09-17 22:36:46.502520 | |
|
||||
| 27 | dpr | opensearch_hnsw | 500000 | 5637 | 477.335 | 11.8093 | 0.0846789 | 93.0814 | 80.8614 | 10 | 2022-09-18 01:53:29.735540 | |
|
||||
@ -1,67 +0,0 @@
|
||||
"""
|
||||
This script performs the same query benchmarking as `retriever.py` but with less of the loops that iterate
|
||||
over all the parameters so that it is easier to inspect what is happening
|
||||
"""
|
||||
|
||||
|
||||
from haystack.document_stores import FAISSDocumentStore
|
||||
from haystack.nodes import DensePassageRetriever
|
||||
from retriever import prepare_data
|
||||
import datetime
|
||||
from pprint import pprint
|
||||
from utils import get_document_store
|
||||
|
||||
|
||||
def benchmark_querying(index_type, n_docs=100_000, similarity="dot_product"):
|
||||
doc_index = "document"
|
||||
label_index = "label"
|
||||
|
||||
docs, labels = prepare_data(
|
||||
data_dir="data/",
|
||||
filename_gold="nq2squad-dev.json",
|
||||
filename_negative="psgs_w100_minus_gold_100k.tsv",
|
||||
remote_url="https://ext-haystack-retriever-eval.s3-eu-west-1.amazonaws.com/",
|
||||
embeddings_filenames=["wikipedia_passages_100k.pkl"],
|
||||
embeddings_dir="embeddings/",
|
||||
n_docs=n_docs,
|
||||
add_precomputed=True,
|
||||
)
|
||||
|
||||
doc_store = get_document_store(document_store_type=index_type, similarity=similarity)
|
||||
|
||||
doc_store.write_documents(documents=docs, index=doc_index)
|
||||
doc_store.write_labels(labels=labels, index=label_index)
|
||||
|
||||
retriever = DensePassageRetriever(
|
||||
document_store=doc_store,
|
||||
query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
|
||||
passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
|
||||
use_gpu=True,
|
||||
use_fast_tokenizers=True,
|
||||
)
|
||||
|
||||
raw_results = retriever.eval(label_index=label_index, doc_index=doc_index)
|
||||
results = {
|
||||
"n_queries": raw_results["n_questions"],
|
||||
"retrieve_time": raw_results["retrieve_time"],
|
||||
"queries_per_second": raw_results["n_questions"] / raw_results["retrieve_time"],
|
||||
"seconds_per_query": raw_results["retrieve_time"] / raw_results["n_questions"],
|
||||
"recall": raw_results["recall"] * 100,
|
||||
"map": raw_results["map"] * 100,
|
||||
"top_k": raw_results["top_k"],
|
||||
"date_time": datetime.datetime.now(),
|
||||
"error": None,
|
||||
}
|
||||
|
||||
pprint(results)
|
||||
|
||||
doc_store.delete_all_documents(index=doc_index)
|
||||
doc_store.delete_all_documents(index=label_index)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
similarity = "l2"
|
||||
n_docs = 1000
|
||||
|
||||
benchmark_querying(index_type="faiss_flat", similarity=similarity, n_docs=n_docs)
|
||||
benchmark_querying(index_type="faiss_hnsw", similarity=similarity, n_docs=n_docs)
|
||||
Loading…
x
Reference in New Issue
Block a user