haystack/docs/v0.5.0/_src/benchmarks/retriever_map.json

{
  "chart_type": "LineChart",
  "title": "Retriever Accuracy",
  "subtitle": "mAP at different number of docs",
  "description": "Here you can see how the mean avg. precision (mAP) of the retriever decays as the number of documents increases. The set up is the same as the above querying benchmark except that a varying number of negative documents are used to fill the document store.",
  "columns": [
    "n_docs",
    "BM25 / ElasticSearch",
    "DPR / ElasticSearch",
    "DPR / FAISS (flat)",
    "DPR / FAISS (HSNW)"
  ],
  "axis": [
     { "x": "Number of docs", "y": "mAP" }
   ],
  "data": [
    {
        "model": "DPR / ElasticSearch",
        "n_docs": 1000,
        "map": 0.929
    },
    {
        "model": "DPR / ElasticSearch",
        "n_docs": 10000,
        "map": 0.898
    },
    {
        "model": "DPR / ElasticSearch",
        "n_docs": 100000,
        "map": 0.863
    },
    {
        "model": "DPR / ElasticSearch",
        "n_docs": 500000,
        "map": 0.805
    },
    {
        "model": "DPR / FAISS (flat)",
        "n_docs": 1000,
        "map": 0.929
    },
    {
        "model": "DPR / FAISS (flat)",
        "n_docs": 10000,
        "map": 0.898
    },
    {
        "model": "DPR / FAISS (flat)",
        "n_docs": 100000,
        "map": 0.863
    },
    {
        "model": "DPR / FAISS (flat)",
        "n_docs": 500000,
        "map": 0.805
    },
    {
        "model": "BM25 / ElasticSearch",
        "n_docs": 1000,
        "map": 0.748
    },
    {
        "model": "BM25 / ElasticSearch",
        "n_docs": 10000,
        "map": 0.6609999999999999
    },
    {
        "model": "BM25 / ElasticSearch",
        "n_docs": 100000,
        "map": 0.56
    },
    {
        "model": "BM25 / ElasticSearch",
        "n_docs": 500000,
        "map": 0.452
    },
    {
        "model": "DPR / FAISS (HSNW)",
        "n_docs": 1000,
        "map": 0.929
    },
    {
        "model": "DPR / FAISS (HSNW)",
        "n_docs": 10000,
        "map": 0.8959999999999999
    },
    {
        "model": "DPR / FAISS (HSNW)",
        "n_docs": 100000,
        "map": 0.8490000000000001
    },
    {
        "model": "DPR / FAISS (HSNW)",
        "n_docs": 500000,
        "map": 0.7659999999999999
    }
  ]
}