haystack/docs/_src/benchmarks/retriever_speed.json

{
    "chart_type": "LineChart",
    "title": "Retriever Speed",
    "subtitle": "Query Speed at different number of docs",
    "description": "Here you can see how the query speed of different Retriever / DocumentStore combinations scale as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
    "columns": [
        "n_docs",
        "BM25 / Elasticsearch",
        "DPR / Elasticsearch",
        "DPR / FAISS (flat)",
        "DPR / FAISS (HNSW)",
        "DPR / Milvus (flat)",
        "DPR / Milvus (HNSW)",
        "Sentence Transformers / Elasticsearch"
    ],
    "axis": [
        {
            "x": "Number of docs",
            "y": "Queries/sec"
        }
    ],
    "data": [
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 1000,
            "query_speed": 34.22768858415144
        },
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 10000,
            "query_speed": 22.197089725786853
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 10000,
            "query_speed": 127.11481826852273
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 1000,
            "query_speed": 47.51341215808855
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 10000,
            "query_speed": 29.74515869340777
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 1000,
            "query_speed": 42.49634272581313
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 10000,
            "query_speed": 27.684040507849826
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 1000,
            "query_speed": 43.36685860983961
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 10000,
            "query_speed": 41.819147130090286
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 1000,
            "query_speed": 41.12204778755844
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 10000,
            "query_speed": 37.86882443918513
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 1000,
            "query_speed": 41.14803671045185
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 10000,
            "query_speed": 40.072871546542935
        },
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 100000,
            "query_speed": 5.192368815242574
        },
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 500000,
            "query_speed": 1.0337466563959614
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 100000,
            "query_speed": 103.0884393334727
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 500000,
            "query_speed": 78.95037031647355
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 100000,
            "query_speed": 6.385621466857457
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 500000,
            "query_speed": 1.4175454254854258
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 100000,
            "query_speed": 6.6270933964840415
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 500000,
            "query_speed": 1.5394964631878052
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 100000,
            "query_speed": 39.7839528511866
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 500000,
            "query_speed": 39.84177061191119
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 100000,
            "query_speed": 28.57264344960955
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 500000,
            "query_speed": 15.645867393099733
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 100000,
            "query_speed": 38.80526238789059
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 500000,
            "query_speed": 37.15717318924075
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 1000,
            "query_speed": 282.95914917837337
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 1000,
            "query_speed": 29.061163356184426
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 10000,
            "query_speed": 24.834414667596725
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 100000,
            "query_speed": 15.306895223372484
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 1000,
            "query_speed": 29.10621389658101
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 10000,
            "query_speed": 26.92417300437131
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 100000,
            "query_speed": 24.95733865947408
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 500000,
            "query_speed": 11.33271222977541
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 500000,
            "query_speed": 24.13921492357397
        }
    ]
}