From c3e59914da14fa47de5a3945aedc7c77c3ab0cfa Mon Sep 17 00:00:00 2001 From: bogdankostic Date: Thu, 1 Jun 2023 13:59:12 +0200 Subject: [PATCH] refactor: Delete outdated benchmark files (#5008) --- test/benchmarks/config.json | 74 ------------ .../data_scripts/embeddings_slice.py | 49 -------- .../data_scripts/shuffle_passages.py | 21 ---- test/benchmarks/reader_results.csv | 7 -- test/benchmarks/reader_results.md | 8 -- test/benchmarks/results_to_json.py | 106 ------------------ test/benchmarks/retriever_index_results.csv | 33 ------ test/benchmarks/retriever_index_results.md | 34 ------ test/benchmarks/retriever_query_results.csv | 33 ------ test/benchmarks/retriever_query_results.md | 34 ------ test/benchmarks/retriever_simplified.py | 67 ----------- 11 files changed, 466 deletions(-) delete mode 100644 test/benchmarks/config.json delete mode 100644 test/benchmarks/data_scripts/embeddings_slice.py delete mode 100644 test/benchmarks/data_scripts/shuffle_passages.py delete mode 100644 test/benchmarks/reader_results.csv delete mode 100644 test/benchmarks/reader_results.md delete mode 100644 test/benchmarks/results_to_json.py delete mode 100644 test/benchmarks/retriever_index_results.csv delete mode 100644 test/benchmarks/retriever_index_results.md delete mode 100644 test/benchmarks/retriever_query_results.csv delete mode 100644 test/benchmarks/retriever_query_results.md delete mode 100644 test/benchmarks/retriever_simplified.py diff --git a/test/benchmarks/config.json b/test/benchmarks/config.json deleted file mode 100644 index 1c2471520..000000000 --- a/test/benchmarks/config.json +++ /dev/null @@ -1,74 +0,0 @@ -{ - "params": { - "full": { - "retriever_doc_stores": [ - [ - "elastic", - "elasticsearch" - ], - [ - "elastic", - "opensearch_flat" - ], - [ - "dpr", - "opensearch_flat" - ], - [ - "dpr", - "opensearch_hnsw" - ], - [ - "dpr", - "elasticsearch" - ], - [ - "dpr", - "faiss_flat" - ], - [ - "dpr", - "faiss_hnsw" - ], - [ - "sentence_transformers", - "elasticsearch" - ] - ], - "n_docs_options": [ - 1000, - 10000, - 100000, - 500000 - ], - "n_queries": null - }, - "ci": { - "retriever_doc_stores": [ - [ - "elastic", - "elasticsearch" - ] - ], - "n_docs_options": [ - 1000 - ], - "n_queries": 100 - } - }, - "filenames": { - "data_s3_url": "https://ext-haystack-retriever-eval.s3-eu-west-1.amazonaws.com/", - "data_dir": "../../data/retriever/", - "filename_gold": "nq2squad-dev.json", - "filenames_negative": { - "10000": "psgs_w100_minus_gold_10k.tsv", - "100000": "psgs_w100_minus_gold_100k.tsv", - "1000000": "psgs_w100_minus_gold_1m.tsv" - }, - "embeddings_dir": "embeddings/", - "embeddings_filenames": { - "10000": "wikipedia_passages_10k.pkl", - "100000": "wikipedia_passages_100k.pkl", - "1000000": "wikipedia_passages_1m.pkl"} - } -} diff --git a/test/benchmarks/data_scripts/embeddings_slice.py b/test/benchmarks/data_scripts/embeddings_slice.py deleted file mode 100644 index a2c4e108e..000000000 --- a/test/benchmarks/data_scripts/embeddings_slice.py +++ /dev/null @@ -1,49 +0,0 @@ -import pickle -from pathlib import Path -from tqdm import tqdm -import json - -n_passages = 1_000_000 -embeddings_dir = Path("embeddings") -embeddings_filenames = [f"wikipedia_passages_{i}.pkl" for i in range(50)] -neg_passages_filename = "psgs_w100_minus_gold.tsv" -gold_passages_filename = "nq2squad-dev.json" - -# Extract gold passage ids -passage_ids = [] -gold_data = json.load(open(gold_passages_filename))["data"] -for d in gold_data: - for p in d["paragraphs"]: - passage_ids.append(str(p["passage_id"])) -print("gold_ids") -print(len(passage_ids)) -print() - -# Extract neg passage ids -with open(neg_passages_filename) as f: - f.readline() # Ignore column headers - for _ in range(n_passages - len(passage_ids)): - l = f.readline() - passage_ids.append(str(l.split()[0])) -assert len(passage_ids) == len(set(passage_ids)) -assert set([type(x) for x in passage_ids]) == {str} -passage_ids = set(passage_ids) -print("all_ids") -print(len(passage_ids)) -print() - - -# Gather vectors for passages -ret = [] -for ef in tqdm(embeddings_filenames): - curr = pickle.load(open(embeddings_dir / ef, "rb")) - for i, vec in curr: - if i in passage_ids: - ret.append((i, vec)) -print("n_vectors") -print(len(ret)) -print() - -# Write vectors to file -with open(f"wikipedia_passages_{n_passages}.pkl", "wb") as f: - pickle.dump(ret, f) diff --git a/test/benchmarks/data_scripts/shuffle_passages.py b/test/benchmarks/data_scripts/shuffle_passages.py deleted file mode 100644 index 3a7808f61..000000000 --- a/test/benchmarks/data_scripts/shuffle_passages.py +++ /dev/null @@ -1,21 +0,0 @@ -import json -from tqdm import tqdm -import time -import random - -random.seed(42) - -lines = [] -with open("psgs_w100_minus_gold_unshuffled.tsv") as f: - f.readline() # Remove column header - lines = [l for l in tqdm(f)] - -tic = time.perf_counter() -random.shuffle(lines) -toc = time.perf_counter() -t = toc - tic -print(t) -with open("psgs_w100_minus_gold.tsv", "w") as f: - f.write("id\ttext\title\n") - for l in tqdm(lines): - f.write(l) diff --git a/test/benchmarks/reader_results.csv b/test/benchmarks/reader_results.csv deleted file mode 100644 index c26dd35ed..000000000 --- a/test/benchmarks/reader_results.csv +++ /dev/null @@ -1,7 +0,0 @@ -,EM,f1,top_n_accuracy,top_n,reader_time,seconds_per_query,EM_text_answer,f1_text_answer,top_n_accuracy_text_answer,top_n_EM_text_answer,top_n_f1_text_answer,Total_text_answer,EM_no_answer,f1_no_answer,top_n_accuracy_no_answer,Total_no_answer,passages_per_second,reader,error -0,76.1672004045171,78.21542223321491,97.41277599865161,5,104.290868674987,0.0087890501158762,61.491731353358084,65.59301387433817,94.81943975700304,81.01586230172123,90.50667071721769,5926,90.8080808080808,90.8080808080808,100.0,5940,118.41880460779026,deepset/roberta-base-squad2, -1,76.84982302376538,79.6479775678566,97.21894488454407,5,52.54729196999688,0.004428391367773207,65.89605129935876,71.49897094501964,94.4313196085049,79.96962537968275,89.70802565150441,5926,87.77777777777777,87.77777777777777,100.0,5940,235.02638360605766,deepset/minilm-uncased-squad2, -2,70.0825889094893,73.49806294434092,95.53345693578291,5,105.55706145399017,0.008895757749367114,56.59804252446844,63.43705955071704,91.05636179547756,72.1228484643942,84.47853540832811,5926,83.53535353535354,83.53535353535354,100.0,5940,116.99833085428466,deepset/bert-base-cased-squad2, -3,83.69290409573573,87.35038229973254,98.07854373841226,5,284.2086430560157,0.023951512140233923,75.19406007424907,82.51765716649112,96.15254809314884,82.33209584880188,91.89205932239825,5926,92.17171717171718,92.17171717171718,100.0,5940,43.45399164221017,deepset/roberta-large-squad2, -4,79.50446654306421,82.29388851439239,97.52233271532108,5,103.88512606802396,0.008754856402159445,71.110361120486,76.69579499017553,95.03881201484981,81.72460344245697,90.89098996208568,5926,87.87878787878788,87.87878787878788,100.0,5940,118.88131118898794,deepset/roberta-base-squad2-distilled, -5,38.53868194842407,43.340283031558755,95.43232765885725,5,62.23228176299017,0.005244588046771462,74.53594330070874,84.1504891077415,90.85386432669593,75.09281133985826,85.21259442024737,5926,2.6262626262626263,2.6262626262626263,100.0,5940,198.4500591997352,distilbert-base-uncased-distilled-squad, diff --git a/test/benchmarks/reader_results.md b/test/benchmarks/reader_results.md deleted file mode 100644 index 7ffe7f7b9..000000000 --- a/test/benchmarks/reader_results.md +++ /dev/null @@ -1,8 +0,0 @@ -| | EM | f1 | top_n_accuracy | top_n | reader_time | seconds_per_query | EM_text_answer | f1_text_answer | top_n_accuracy_text_answer | top_n_EM_text_answer | top_n_f1_text_answer | Total_text_answer | EM_no_answer | f1_no_answer | top_n_accuracy_no_answer | Total_no_answer | passages_per_second | reader | error | -|---:|--------:|--------:|-----------------:|--------:|--------------:|--------------------:|-----------------:|-----------------:|-----------------------------:|-----------------------:|-----------------------:|--------------------:|---------------:|---------------:|---------------------------:|------------------:|----------------------:|:----------------------------------------|:--------| -| 0 | 76.1672 | 78.2154 | 97.4128 | 5 | 104.291 | 0.00878905 | 61.4917 | 65.593 | 94.8194 | 81.0159 | 90.5067 | 5926 | 90.8081 | 90.8081 | 100 | 5940 | 118.419 | deepset/roberta-base-squad2 | | -| 1 | 76.8498 | 79.648 | 97.2189 | 5 | 52.5473 | 0.00442839 | 65.8961 | 71.499 | 94.4313 | 79.9696 | 89.708 | 5926 | 87.7778 | 87.7778 | 100 | 5940 | 235.026 | deepset/minilm-uncased-squad2 | | -| 2 | 70.0826 | 73.4981 | 95.5335 | 5 | 105.557 | 0.00889576 | 56.598 | 63.4371 | 91.0564 | 72.1228 | 84.4785 | 5926 | 83.5354 | 83.5354 | 100 | 5940 | 116.998 | deepset/bert-base-cased-squad2 | | -| 3 | 83.6929 | 87.3504 | 98.0785 | 5 | 284.209 | 0.0239515 | 75.1941 | 82.5177 | 96.1525 | 82.3321 | 91.8921 | 5926 | 92.1717 | 92.1717 | 100 | 5940 | 43.454 | deepset/roberta-large-squad2 | | -| 4 | 79.5045 | 82.2939 | 97.5223 | 5 | 103.885 | 0.00875486 | 71.1104 | 76.6958 | 95.0388 | 81.7246 | 90.891 | 5926 | 87.8788 | 87.8788 | 100 | 5940 | 118.881 | deepset/roberta-base-squad2-distilled | | -| 5 | 38.5387 | 43.3403 | 95.4323 | 5 | 62.2323 | 0.00524459 | 74.5359 | 84.1505 | 90.8539 | 75.0928 | 85.2126 | 5926 | 2.62626 | 2.62626 | 100 | 5940 | 198.45 | distilbert-base-uncased-distilled-squad | | diff --git a/test/benchmarks/results_to_json.py b/test/benchmarks/results_to_json.py deleted file mode 100644 index 5f06d612c..000000000 --- a/test/benchmarks/results_to_json.py +++ /dev/null @@ -1,106 +0,0 @@ -import json -import pandas as pd -from pprint import pprint - - -def reader(reader_csv="reader_results.csv"): - model_rename_map = { - "deepset/roberta-base-squad2": "RoBERTa", - "deepset/minilm-uncased-squad2": "MiniLM", - "deepset/bert-base-cased-squad2": "BERT base", - "deepset/bert-large-uncased-whole-word-masking-squad2": "BERT large", - "deepset/xlm-roberta-large-squad2": "XLM-RoBERTa", - "deepset/roberta-large-squad2": "RoBERTa large", - "deepset/roberta-base-squad2-distilled": "RoBERTa distilled", - "distilbert-base-uncased-distilled-squad": "distilBERT", - } - - column_name_map = {"f1": "F1", "passages_per_second": "Speed", "reader": "Model"} - - df = pd.read_csv(reader_csv) - df = df[["f1", "passages_per_second", "reader"]] - df["reader"] = df["reader"].map(model_rename_map) - df = df[list(column_name_map)] - df = df.rename(columns=column_name_map) - ret = [dict(row) for i, row in df.iterrows()] - print("Reader overview") - print(json.dumps(ret, indent=4)) - return ret - - -def retriever(index_csv="retriever_index_results.csv", query_csv="retriever_query_results.csv"): - column_name_map = { - "model": "model", - "n_docs": "n_docs", - "docs_per_second": "index_speed", - "queries_per_second": "query_speed", - "map": "map", - } - - name_cleaning = { - "dpr": "DPR", - "elastic": "BM25", - "elasticsearch": "Elasticsearch", - "faiss": "FAISS", - "faiss_flat": "FAISS (flat)", - "faiss_hnsw": "FAISS (HNSW)", - "sentence_transformers": "Sentence Transformers", - "opensearch_flat": "OpenSearch (flat)", - "opensearch_hnsw": "OpenSearch (HNSW)", - } - - index = pd.read_csv(index_csv) - query = pd.read_csv(query_csv) - df = pd.merge( - index, - query, - how="right", - left_on=["retriever", "doc_store", "n_docs"], - right_on=["retriever", "doc_store", "n_docs"], - ) - - df["retriever"] = df["retriever"].map(name_cleaning) - df["doc_store"] = df["doc_store"].map(name_cleaning) - df["model"] = df["retriever"] + " / " + df["doc_store"] - - df = df[list(column_name_map)] - df = df.rename(columns=column_name_map) - - print("Retriever overview") - retriever_overview_data = retriever_overview(df) - print(json.dumps(retriever_overview_data, indent=4)) - - print("Retriever MAP") - retriever_map_data = retriever_map(df) - print(json.dumps(retriever_map_data, indent=4)) - - print("Retriever Speed") - retriever_speed_data = retriever_speed(df) - print(json.dumps(retriever_speed_data, indent=4)) - - return retriever_overview_data, retriever_map_data, retriever_speed_data - - -def retriever_map(df): - columns = ["model", "n_docs", "map"] - df = df[columns] - ret = df.to_dict(orient="records") - return ret - - -def retriever_speed(df): - columns = ["model", "n_docs", "query_speed"] - df = df[columns] - ret = df.to_dict(orient="records") - return ret - - -def retriever_overview(df, chosen_n_docs=100_000): - df = df[df["n_docs"] == chosen_n_docs] - ret = [dict(row) for i, row in df.iterrows()] - return ret - - -if __name__ == "__main__": - reader() - retriever() diff --git a/test/benchmarks/retriever_index_results.csv b/test/benchmarks/retriever_index_results.csv deleted file mode 100644 index 88b5051fb..000000000 --- a/test/benchmarks/retriever_index_results.csv +++ /dev/null @@ -1,33 +0,0 @@ -,retriever,doc_store,n_docs,indexing_time,docs_per_second,date_time,error -31,sentence_transformers,elasticsearch,500000,4165.329931593005,120.03851032486591,2022-09-17 21:16:26.676611, -28,dpr,elasticsearch,500000,6180.3500814459985,80.90156599721556,2022-09-17 16:45:36.329915, -7,sentence_transformers,elasticsearch,1000,9.844315000998904,101.5814711230319,2022-09-17 08:26:44.750140, -15,sentence_transformers,elasticsearch,10000,84.88162445399212,117.8111289024663,2022-09-17 08:43:51.615496, -20,dpr,elasticsearch,100000,1230.761674939,81.2504988059173,2022-09-17 09:57:09.456572, -0,elastic,elasticsearch,1000,1.7988810280075995,555.9011321096525,2022-09-17 08:21:11.001120, -24,elastic,elasticsearch,500000,1031.7205082759901,484.6273733915616,2022-09-17 11:09:14.536450, -16,elastic,elasticsearch,100000,206.83811076599522,483.4698964792531,2022-09-17 08:47:59.723168, -8,elastic,elasticsearch,10000,20.81922616600059,480.3252493760203,2022-09-17 08:27:43.460016, -4,dpr,elasticsearch,1000,12.114610354998149,82.54495775734364,2022-09-17 08:24:34.582262, -12,dpr,elasticsearch,10000,123.21874477900565,81.15648327642944,2022-09-17 08:37:02.872786, -23,sentence_transformers,elasticsearch,100000,831.661083732004,120.24128813537725,2022-09-17 10:50:54.888143, -29,dpr,faiss_flat,500000,5485.974153257004,91.1415158059306,2022-09-17 18:19:50.024476, -13,dpr,faiss_flat,10000,105.0845138200093,95.16150036272884,2022-09-17 08:39:25.198922, -5,dpr,faiss_flat,1000,10.151226033005514,98.51026829159531,2022-09-17 08:25:18.819509, -21,dpr,faiss_flat,100000,1068.483952116003,93.59054930302146,2022-09-17 10:15:58.178598, -22,dpr,faiss_hnsw,100000,1195.8989856069966,83.61910261947709,2022-09-17 10:36:24.783587, -14,dpr,faiss_hnsw,10000,114.6786555860017,87.20018515128682,2022-09-17 08:41:49.541725, -6,dpr,faiss_hnsw,1000,10.710903742990922,93.36280336329109,2022-09-17 08:25:58.505307, -30,dpr,faiss_hnsw,500000,6349.903588575005,78.7413530025274,2022-09-17 20:06:17.374842, -9,elastic,opensearch_flat,10000,21.9185139649926,456.2353093814486,2022-09-17 08:28:40.051299, -10,dpr,opensearch_flat,10000,123.36043137000524,81.06327036102982,2022-09-17 08:31:26.201876, -17,elastic,opensearch_flat,100000,208.89336037798785,478.7131568904452,2022-09-17 08:52:13.178937, -1,elastic,opensearch_flat,1000,2.2300678319879808,448.4168533602657,2022-09-17 08:21:45.610495, -18,dpr,opensearch_flat,100000,1245.0073385680007,80.32081169498916,2022-09-17 09:13:46.979550, -25,elastic,opensearch_flat,500000,1045.8889810429973,478.06221220667453,2022-09-17 11:28:08.906406, -26,dpr,opensearch_flat,500000,6235.606822678994,80.18465791997863,2022-09-17 13:13:24.174105, -2,dpr,opensearch_flat,1000,12.919973790005315,77.3995378205476,2022-09-17 08:22:43.256934, -11,dpr,opensearch_hnsw,10000,121.49061479599914,82.3108848102505,2022-09-17 08:34:11.508717, -3,dpr,opensearch_hnsw,1000,12.355953149002744,80.93264744053441,2022-09-17 08:23:36.852546, -19,dpr,opensearch_hnsw,100000,1242.960266391994,80.45309468360983,2022-09-17 09:35:31.609351, -27,dpr,opensearch_hnsw,500000,6237.192302857991,80.16427516125985,2022-09-17 14:59:57.763007, diff --git a/test/benchmarks/retriever_index_results.md b/test/benchmarks/retriever_index_results.md deleted file mode 100644 index 03e0a4244..000000000 --- a/test/benchmarks/retriever_index_results.md +++ /dev/null @@ -1,34 +0,0 @@ -| | retriever | doc_store | n_docs | indexing_time | docs_per_second | date_time | error | -|---:|:----------------------|:----------------|---------:|----------------:|------------------:|:---------------------------|:--------| -| 31 | sentence_transformers | elasticsearch | 500000 | 4165.33 | 120.039 | 2022-09-17 21:16:26.676611 | | -| 28 | dpr | elasticsearch | 500000 | 6180.35 | 80.9016 | 2022-09-17 16:45:36.329915 | | -| 7 | sentence_transformers | elasticsearch | 1000 | 9.84432 | 101.581 | 2022-09-17 08:26:44.750140 | | -| 15 | sentence_transformers | elasticsearch | 10000 | 84.8816 | 117.811 | 2022-09-17 08:43:51.615496 | | -| 20 | dpr | elasticsearch | 100000 | 1230.76 | 81.2505 | 2022-09-17 09:57:09.456572 | | -| 0 | elastic | elasticsearch | 1000 | 1.79888 | 555.901 | 2022-09-17 08:21:11.001120 | | -| 24 | elastic | elasticsearch | 500000 | 1031.72 | 484.627 | 2022-09-17 11:09:14.536450 | | -| 16 | elastic | elasticsearch | 100000 | 206.838 | 483.47 | 2022-09-17 08:47:59.723168 | | -| 8 | elastic | elasticsearch | 10000 | 20.8192 | 480.325 | 2022-09-17 08:27:43.460016 | | -| 4 | dpr | elasticsearch | 1000 | 12.1146 | 82.545 | 2022-09-17 08:24:34.582262 | | -| 12 | dpr | elasticsearch | 10000 | 123.219 | 81.1565 | 2022-09-17 08:37:02.872786 | | -| 23 | sentence_transformers | elasticsearch | 100000 | 831.661 | 120.241 | 2022-09-17 10:50:54.888143 | | -| 29 | dpr | faiss_flat | 500000 | 5485.97 | 91.1415 | 2022-09-17 18:19:50.024476 | | -| 13 | dpr | faiss_flat | 10000 | 105.085 | 95.1615 | 2022-09-17 08:39:25.198922 | | -| 5 | dpr | faiss_flat | 1000 | 10.1512 | 98.5103 | 2022-09-17 08:25:18.819509 | | -| 21 | dpr | faiss_flat | 100000 | 1068.48 | 93.5905 | 2022-09-17 10:15:58.178598 | | -| 22 | dpr | faiss_hnsw | 100000 | 1195.9 | 83.6191 | 2022-09-17 10:36:24.783587 | | -| 14 | dpr | faiss_hnsw | 10000 | 114.679 | 87.2002 | 2022-09-17 08:41:49.541725 | | -| 6 | dpr | faiss_hnsw | 1000 | 10.7109 | 93.3628 | 2022-09-17 08:25:58.505307 | | -| 30 | dpr | faiss_hnsw | 500000 | 6349.9 | 78.7414 | 2022-09-17 20:06:17.374842 | | -| 9 | elastic | opensearch_flat | 10000 | 21.9185 | 456.235 | 2022-09-17 08:28:40.051299 | | -| 10 | dpr | opensearch_flat | 10000 | 123.36 | 81.0633 | 2022-09-17 08:31:26.201876 | | -| 17 | elastic | opensearch_flat | 100000 | 208.893 | 478.713 | 2022-09-17 08:52:13.178937 | | -| 1 | elastic | opensearch_flat | 1000 | 2.23007 | 448.417 | 2022-09-17 08:21:45.610495 | | -| 18 | dpr | opensearch_flat | 100000 | 1245.01 | 80.3208 | 2022-09-17 09:13:46.979550 | | -| 25 | elastic | opensearch_flat | 500000 | 1045.89 | 478.062 | 2022-09-17 11:28:08.906406 | | -| 26 | dpr | opensearch_flat | 500000 | 6235.61 | 80.1847 | 2022-09-17 13:13:24.174105 | | -| 2 | dpr | opensearch_flat | 1000 | 12.92 | 77.3995 | 2022-09-17 08:22:43.256934 | | -| 11 | dpr | opensearch_hnsw | 10000 | 121.491 | 82.3109 | 2022-09-17 08:34:11.508717 | | -| 3 | dpr | opensearch_hnsw | 1000 | 12.356 | 80.9326 | 2022-09-17 08:23:36.852546 | | -| 19 | dpr | opensearch_hnsw | 100000 | 1242.96 | 80.4531 | 2022-09-17 09:35:31.609351 | | -| 27 | dpr | opensearch_hnsw | 500000 | 6237.19 | 80.1643 | 2022-09-17 14:59:57.763007 | | diff --git a/test/benchmarks/retriever_query_results.csv b/test/benchmarks/retriever_query_results.csv deleted file mode 100644 index 73febb396..000000000 --- a/test/benchmarks/retriever_query_results.csv +++ /dev/null @@ -1,33 +0,0 @@ -,retriever,doc_store,n_docs,n_queries,retrieve_time,queries_per_second,seconds_per_query,recall,map,top_k,date_time,error -31,sentence_transformers,elasticsearch,500000,5637,3910.198875275557,1.4416146543448518,0.6936666445406345,89.6398793684584,76.49564526892908,10,2022-09-18 08:25:36.627368, -28,dpr,elasticsearch,500000,5637,3839.528848723392,1.4681488854743858,0.6811298294701778,93.0814262905801,80.86137228234092,10,2022-09-18 03:20:53.663119, -7,sentence_transformers,elasticsearch,1000,1064,22.08566212869482,48.176051675516526,0.020757201248773328,98.7781954887218,90.06638620360428,10,2022-09-17 21:28:29.461918, -15,sentence_transformers,elasticsearch,10000,5637,187.41362470001332,30.0778559137467,0.03324705068298977,96.52297321270179,87.11255142468552,10,2022-09-17 22:03:32.200427, -20,dpr,elasticsearch,100000,5637,903.5089274104685,6.239008635095737,0.16028187465149343,95.7956359765833,86.5456409043424,10,2022-09-17 22:58:20.213034, -0,elastic,elasticsearch,1000,1064,4.122031082282774,258.12517634164914,0.00387408936304772,89.09774436090225,74.2044471297291,10,2022-09-17 21:19:21.768721, -24,elastic,elasticsearch,500000,5637,83.56125180030358,67.45949681882962,0.014823709739276845,62.746141564662054,45.603960228760656,10,2022-09-18 00:29:07.528766, -16,elastic,elasticsearch,100000,5637,42.696119271276984,132.02605052193084,0.007574262776525986,71.93542664537875,56.26106985872818,10,2022-09-17 22:08:40.215954, -8,elastic,elasticsearch,10000,5637,28.68551177883637,196.51035140878582,0.005088790452161854,81.10697179350718,66.32941083712781,10,2022-09-17 21:30:14.745163, -4,dpr,elasticsearch,1000,1064,29.50371559121413,36.06325436233689,0.0277290560067802,99.15413533834587,92.9510532283089,10,2022-09-17 21:24:50.768859, -12,dpr,elasticsearch,10000,5637,224.79332777825766,25.076367059971158,0.03987818481076063,97.49866950505589,89.87097014904354,10,2022-09-17 21:48:36.651933, -23,sentence_transformers,elasticsearch,100000,5637,886.5846678019152,6.358106794217024,0.15727952240587462,93.79102359411034,82.74686664920834,10,2022-09-18 00:09:04.840841, -29,dpr,faiss_flat,500000,5637,3683.516663164017,1.5303310709494677,0.6534533729224795,93.0814262905801,80.86137228234091,10,2022-09-18 05:09:24.992364, -13,dpr,faiss_flat,10000,5637,181.46477086472441,31.06388073640026,0.032191728022835625,97.49866950505589,89.87097014904354,10,2022-09-17 21:53:56.820064, -5,dpr,faiss_flat,1000,1064,29.979966158716707,35.49036694594937,0.028176659923605928,99.15413533834587,92.9510532283089,10,2022-09-17 21:26:11.004926, -21,dpr,faiss_flat,100000,5637,841.2282260610373,6.700916380795566,0.14923332021661118,95.7956359765833,86.54606328368972,10,2022-09-17 23:23:47.065373, -22,dpr,faiss_hnsw,100000,5637,138.64402735611657,40.6580803190388,0.024595356990618513,93.18786588610963,84.33419639513305,10,2022-09-17 23:39:25.396237, -14,dpr,faiss_hnsw,10000,5637,112.71057137736352,50.01305495228924,0.019994779382182637,96.98421145999644,89.51337675393017,10,2022-09-17 21:58:05.961829, -6,dpr,faiss_hnsw,1000,1064,25.61499380285386,41.538171283159,0.02407424229591528,99.15413533834587,92.9510532283089,10,2022-09-17 21:27:21.948745, -30,dpr,faiss_hnsw,500000,5637,124.99580949536175,45.097511850660666,0.022174172342622273,86.90792974986695,75.73315903145605,10,2022-09-18 06:09:37.007518, -9,elastic,opensearch_flat,10000,5637,32.42948475538287,173.8232982275283,0.005752968734323731,81.10697179350718,66.32941083712774,10,2022-09-17 21:32:03.707902, -10,dpr,opensearch_flat,10000,5637,182.6267182478914,30.86623936563611,0.03239785670532046,97.49866950505589,89.87097014904354,10,2022-09-17 21:37:08.044354, -17,elastic,opensearch_flat,100000,5637,46.926014778233366,120.12526583899732,0.008324643388013725,71.93542664537875,56.26106985872818,10,2022-09-17 22:13:58.492431, -1,elastic,opensearch_flat,1000,1064,6.101849522849079,174.3733594241761,0.005734820980121315,89.09774436090225,74.2044471297291,10,2022-09-17 21:20:04.003708, -18,dpr,opensearch_flat,100000,5637,320.2046442734136,17.604366772353018,0.056804088038569026,95.76015611140679,86.51945338480733,10,2022-09-17 22:25:11.379134, -25,elastic,opensearch_flat,500000,5637,83.39615098570357,67.59304756122785,0.014794420966064143,62.746141564662054,45.59509026246651,10,2022-09-18 00:49:52.903674, -26,dpr,opensearch_flat,500000,5637,466.1570336711302,12.092491570076481,0.08269594352867309,93.0814262905801,80.861372282341,10,2022-09-18 01:20:53.923798, -2,dpr,opensearch_flat,1000,1064,30.094220374303404,35.35562598951788,0.02828404170517237,99.15413533834587,92.9510532283089,10,2022-09-17 21:21:42.413924, -11,dpr,opensearch_hnsw,10000,5637,181.91717168685864,30.986629506879073,0.032271983623710955,97.49866950505589,89.87097014904357,10,2022-09-17 21:42:16.601719, -3,dpr,opensearch_hnsw,1000,1064,29.82644817981054,35.67303735213834,0.028032376108844494,99.15413533834587,92.9510532283089,10,2022-09-17 21:23:12.487039, -19,dpr,opensearch_hnsw,100000,5637,327.2236303977261,17.226750993345046,0.058049251445401115,95.77789604399504,86.53719331739565,10,2022-09-17 22:36:46.502520, -27,dpr,opensearch_hnsw,500000,5637,477.3347775057773,11.809321812785313,0.08467886774982744,93.0814262905801,80.861372282341,10,2022-09-18 01:53:29.735540, diff --git a/test/benchmarks/retriever_query_results.md b/test/benchmarks/retriever_query_results.md deleted file mode 100644 index e6b20a9ba..000000000 --- a/test/benchmarks/retriever_query_results.md +++ /dev/null @@ -1,34 +0,0 @@ -| | retriever | doc_store | n_docs | n_queries | retrieve_time | queries_per_second | seconds_per_query | recall | map | top_k | date_time | error | -|---:|:----------------------|:----------------|---------:|------------:|----------------:|---------------------:|--------------------:|---------:|--------:|--------:|:---------------------------|:--------| -| 31 | sentence_transformers | elasticsearch | 500000 | 5637 | 3910.2 | 1.44161 | 0.693667 | 89.6399 | 76.4956 | 10 | 2022-09-18 08:25:36.627368 | | -| 28 | dpr | elasticsearch | 500000 | 5637 | 3839.53 | 1.46815 | 0.68113 | 93.0814 | 80.8614 | 10 | 2022-09-18 03:20:53.663119 | | -| 7 | sentence_transformers | elasticsearch | 1000 | 1064 | 22.0857 | 48.1761 | 0.0207572 | 98.7782 | 90.0664 | 10 | 2022-09-17 21:28:29.461918 | | -| 15 | sentence_transformers | elasticsearch | 10000 | 5637 | 187.414 | 30.0779 | 0.0332471 | 96.523 | 87.1126 | 10 | 2022-09-17 22:03:32.200427 | | -| 20 | dpr | elasticsearch | 100000 | 5637 | 903.509 | 6.23901 | 0.160282 | 95.7956 | 86.5456 | 10 | 2022-09-17 22:58:20.213034 | | -| 0 | elastic | elasticsearch | 1000 | 1064 | 4.12203 | 258.125 | 0.00387409 | 89.0977 | 74.2044 | 10 | 2022-09-17 21:19:21.768721 | | -| 24 | elastic | elasticsearch | 500000 | 5637 | 83.5613 | 67.4595 | 0.0148237 | 62.7461 | 45.604 | 10 | 2022-09-18 00:29:07.528766 | | -| 16 | elastic | elasticsearch | 100000 | 5637 | 42.6961 | 132.026 | 0.00757426 | 71.9354 | 56.2611 | 10 | 2022-09-17 22:08:40.215954 | | -| 8 | elastic | elasticsearch | 10000 | 5637 | 28.6855 | 196.51 | 0.00508879 | 81.107 | 66.3294 | 10 | 2022-09-17 21:30:14.745163 | | -| 4 | dpr | elasticsearch | 1000 | 1064 | 29.5037 | 36.0633 | 0.0277291 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:24:50.768859 | | -| 12 | dpr | elasticsearch | 10000 | 5637 | 224.793 | 25.0764 | 0.0398782 | 97.4987 | 89.871 | 10 | 2022-09-17 21:48:36.651933 | | -| 23 | sentence_transformers | elasticsearch | 100000 | 5637 | 886.585 | 6.35811 | 0.15728 | 93.791 | 82.7469 | 10 | 2022-09-18 00:09:04.840841 | | -| 29 | dpr | faiss_flat | 500000 | 5637 | 3683.52 | 1.53033 | 0.653453 | 93.0814 | 80.8614 | 10 | 2022-09-18 05:09:24.992364 | | -| 13 | dpr | faiss_flat | 10000 | 5637 | 181.465 | 31.0639 | 0.0321917 | 97.4987 | 89.871 | 10 | 2022-09-17 21:53:56.820064 | | -| 5 | dpr | faiss_flat | 1000 | 1064 | 29.98 | 35.4904 | 0.0281767 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:26:11.004926 | | -| 21 | dpr | faiss_flat | 100000 | 5637 | 841.228 | 6.70092 | 0.149233 | 95.7956 | 86.5461 | 10 | 2022-09-17 23:23:47.065373 | | -| 22 | dpr | faiss_hnsw | 100000 | 5637 | 138.644 | 40.6581 | 0.0245954 | 93.1879 | 84.3342 | 10 | 2022-09-17 23:39:25.396237 | | -| 14 | dpr | faiss_hnsw | 10000 | 5637 | 112.711 | 50.0131 | 0.0199948 | 96.9842 | 89.5134 | 10 | 2022-09-17 21:58:05.961829 | | -| 6 | dpr | faiss_hnsw | 1000 | 1064 | 25.615 | 41.5382 | 0.0240742 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:27:21.948745 | | -| 30 | dpr | faiss_hnsw | 500000 | 5637 | 124.996 | 45.0975 | 0.0221742 | 86.9079 | 75.7332 | 10 | 2022-09-18 06:09:37.007518 | | -| 9 | elastic | opensearch_flat | 10000 | 5637 | 32.4295 | 173.823 | 0.00575297 | 81.107 | 66.3294 | 10 | 2022-09-17 21:32:03.707902 | | -| 10 | dpr | opensearch_flat | 10000 | 5637 | 182.627 | 30.8662 | 0.0323979 | 97.4987 | 89.871 | 10 | 2022-09-17 21:37:08.044354 | | -| 17 | elastic | opensearch_flat | 100000 | 5637 | 46.926 | 120.125 | 0.00832464 | 71.9354 | 56.2611 | 10 | 2022-09-17 22:13:58.492431 | | -| 1 | elastic | opensearch_flat | 1000 | 1064 | 6.10185 | 174.373 | 0.00573482 | 89.0977 | 74.2044 | 10 | 2022-09-17 21:20:04.003708 | | -| 18 | dpr | opensearch_flat | 100000 | 5637 | 320.205 | 17.6044 | 0.0568041 | 95.7602 | 86.5195 | 10 | 2022-09-17 22:25:11.379134 | | -| 25 | elastic | opensearch_flat | 500000 | 5637 | 83.3962 | 67.593 | 0.0147944 | 62.7461 | 45.5951 | 10 | 2022-09-18 00:49:52.903674 | | -| 26 | dpr | opensearch_flat | 500000 | 5637 | 466.157 | 12.0925 | 0.0826959 | 93.0814 | 80.8614 | 10 | 2022-09-18 01:20:53.923798 | | -| 2 | dpr | opensearch_flat | 1000 | 1064 | 30.0942 | 35.3556 | 0.028284 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:21:42.413924 | | -| 11 | dpr | opensearch_hnsw | 10000 | 5637 | 181.917 | 30.9866 | 0.032272 | 97.4987 | 89.871 | 10 | 2022-09-17 21:42:16.601719 | | -| 3 | dpr | opensearch_hnsw | 1000 | 1064 | 29.8264 | 35.673 | 0.0280324 | 99.1541 | 92.9511 | 10 | 2022-09-17 21:23:12.487039 | | -| 19 | dpr | opensearch_hnsw | 100000 | 5637 | 327.224 | 17.2268 | 0.0580493 | 95.7779 | 86.5372 | 10 | 2022-09-17 22:36:46.502520 | | -| 27 | dpr | opensearch_hnsw | 500000 | 5637 | 477.335 | 11.8093 | 0.0846789 | 93.0814 | 80.8614 | 10 | 2022-09-18 01:53:29.735540 | | diff --git a/test/benchmarks/retriever_simplified.py b/test/benchmarks/retriever_simplified.py deleted file mode 100644 index c72fc3414..000000000 --- a/test/benchmarks/retriever_simplified.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -This script performs the same query benchmarking as `retriever.py` but with less of the loops that iterate -over all the parameters so that it is easier to inspect what is happening -""" - - -from haystack.document_stores import FAISSDocumentStore -from haystack.nodes import DensePassageRetriever -from retriever import prepare_data -import datetime -from pprint import pprint -from utils import get_document_store - - -def benchmark_querying(index_type, n_docs=100_000, similarity="dot_product"): - doc_index = "document" - label_index = "label" - - docs, labels = prepare_data( - data_dir="data/", - filename_gold="nq2squad-dev.json", - filename_negative="psgs_w100_minus_gold_100k.tsv", - remote_url="https://ext-haystack-retriever-eval.s3-eu-west-1.amazonaws.com/", - embeddings_filenames=["wikipedia_passages_100k.pkl"], - embeddings_dir="embeddings/", - n_docs=n_docs, - add_precomputed=True, - ) - - doc_store = get_document_store(document_store_type=index_type, similarity=similarity) - - doc_store.write_documents(documents=docs, index=doc_index) - doc_store.write_labels(labels=labels, index=label_index) - - retriever = DensePassageRetriever( - document_store=doc_store, - query_embedding_model="facebook/dpr-question_encoder-single-nq-base", - passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base", - use_gpu=True, - use_fast_tokenizers=True, - ) - - raw_results = retriever.eval(label_index=label_index, doc_index=doc_index) - results = { - "n_queries": raw_results["n_questions"], - "retrieve_time": raw_results["retrieve_time"], - "queries_per_second": raw_results["n_questions"] / raw_results["retrieve_time"], - "seconds_per_query": raw_results["retrieve_time"] / raw_results["n_questions"], - "recall": raw_results["recall"] * 100, - "map": raw_results["map"] * 100, - "top_k": raw_results["top_k"], - "date_time": datetime.datetime.now(), - "error": None, - } - - pprint(results) - - doc_store.delete_all_documents(index=doc_index) - doc_store.delete_all_documents(index=label_index) - - -if __name__ == "__main__": - similarity = "l2" - n_docs = 1000 - - benchmark_querying(index_type="faiss_flat", similarity=similarity, n_docs=n_docs) - benchmark_querying(index_type="faiss_hnsw", similarity=similarity, n_docs=n_docs)