haystack/docs/v1.1.0/_src/benchmarks/retriever_map.json

{
    "chart_type": "LineChart",
    "title": "Retriever Accuracy",
    "subtitle": "mAP at different number of docs",
    "description": "Here you can see how the mean avg. precision (mAP) of the retriever decays as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
    "columns": [
        "n_docs",
        "BM25 / Elasticsearch",
        "DPR / Elasticsearch",
        "DPR / FAISS (flat)",
        "DPR / FAISS (HNSW)",
        "DPR / Milvus (flat)",
        "DPR / Milvus (HNSW)",
        "Sentence Transformers / Elasticsearch"
    ],
    "axis": [
        {
            "x": "Number of docs",
            "y": "mAP"
        }
    ],
    "data": [
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 1000,
            "map": 92.95105322830891
        },
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 10000,
            "map": 89.87097014904354
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 10000,
            "map": 66.26543444531747
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 1000,
            "map": 90.06638620360428
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 10000,
            "map": 87.11255142468549
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 1000,
            "map": 92.95105322830891
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 10000,
            "map": 89.87097014904354
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 1000,
            "map": 92.95105322830891
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 10000,
            "map": 89.51337675393017
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 1000,
            "map": 92.95105322830891
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 10000,
            "map": 89.87097014904354
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 1000,
            "map": 92.95105322830891
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 10000,
            "map": 88.24421129104469
        },
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 100000,
            "map": 86.54606328368976
        },
        {
            "model": "DPR / Elasticsearch",
            "n_docs": 500000,
            "map": 80.86137228234091
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 100000,
            "map": 56.25299537353825
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 500000,
            "map": 45.595090262466535
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 100000,
            "map": 82.74686664920836
        },
        {
            "model": "Sentence Transformers / Elasticsearch",
            "n_docs": 500000,
            "map": 76.49564526892904
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 100000,
            "map": 86.54606328368973
        },
        {
            "model": "DPR / FAISS (flat)",
            "n_docs": 500000,
            "map": 80.86137228234091
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 100000,
            "map": 84.33419639513305
        },
        {
            "model": "DPR / FAISS (HNSW)",
            "n_docs": 500000,
            "map": 75.73062475537202
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 100000,
            "map": 86.54606328368973
        },
        {
            "model": "DPR / Milvus (flat)",
            "n_docs": 500000,
            "map": 80.86137228234091
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 100000,
            "map": 81.63864883662649
        },
        {
            "model": "DPR / Milvus (HNSW)",
            "n_docs": 500000,
            "map": 73.57986207906387
        },
        {
            "model": "BM25 / Elasticsearch",
            "n_docs": 1000,
            "map": 74.20444712972909
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 1000,
            "map": 92.95105322830891
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 10000,
            "map": 89.8709701490436
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 100000,
            "map": 86.54014997282701
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 1000,
            "map": 92.76308330349686
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 10000,
            "map": 89.00403653862938
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 100000,
            "map": 85.7342431384476
        },
        {
            "model": "DPR / OpenSearch (flat)",
            "n_docs": 500000,
            "map": 80.85588135082547
        },
        {
            "model": "DPR / OpenSearch (HNSW)",
            "n_docs": 500000,
            "map": 77.5426462347698
        }
    ]
}