mirror of
https://github.com/deepset-ai/haystack.git
synced 2025-06-26 22:00:13 +00:00
refactor: Rename DocumentMeanAveragePrecision
and DocumentMeanReciprocalRank
(#7470)
* Rename DocumentMeanAveragePrecision and DocumentMeanReciprocalRank * Update releasenotes * Simplify names
This commit is contained in:
parent
bf8453e48e
commit
8b8a93bc0d
@ -4,20 +4,20 @@ from haystack import Document, component
|
||||
|
||||
|
||||
@component
|
||||
class DocumentMeanAveragePrecision:
|
||||
class DocumentMAPEvaluator:
|
||||
"""
|
||||
Evaluator that calculates the mean average precision of the retrieved documents, a metric
|
||||
that measures how high retrieved documents are ranked.
|
||||
Each question can have multiple ground truth documents and multiple retrieved documents.
|
||||
|
||||
`DocumentMeanAveragePrecision` doesn't normalize its inputs, the `DocumentCleaner` component
|
||||
`DocumentMAPEvaluator` doesn't normalize its inputs, the `DocumentCleaner` component
|
||||
should be used to clean and normalize the documents before passing them to this evaluator.
|
||||
|
||||
Usage example:
|
||||
```python
|
||||
from haystack.components.evaluators import AnswerExactMatchEvaluator
|
||||
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[
|
||||
[Document(content="France")],
|
||||
@ -41,7 +41,7 @@ class DocumentMeanAveragePrecision:
|
||||
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run the DocumentMeanAveragePrecision on the given inputs.
|
||||
Run the DocumentMAPEvaluator on the given inputs.
|
||||
All lists must have the same length.
|
||||
|
||||
:param ground_truth_documents:
|
||||
|
@ -4,20 +4,20 @@ from haystack import Document, component
|
||||
|
||||
|
||||
@component
|
||||
class DocumentMeanReciprocalRank:
|
||||
class DocumentMRREvaluator:
|
||||
"""
|
||||
Evaluator that calculates the mean reciprocal rank of the retrieved documents.
|
||||
|
||||
MRR measures how high the first retrieved document is ranked.
|
||||
Each question can have multiple ground truth documents and multiple retrieved documents.
|
||||
|
||||
`DocumentMeanReciprocalRank` doesn't normalize its inputs, the `DocumentCleaner` component
|
||||
`DocumentMRREvaluator` doesn't normalize its inputs, the `DocumentCleaner` component
|
||||
should be used to clean and normalize the documents before passing them to this evaluator.
|
||||
|
||||
Usage example:
|
||||
```python
|
||||
from haystack.components.evaluators import AnswerExactMatchEvaluator
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[
|
||||
[Document(content="France")],
|
||||
@ -40,7 +40,7 @@ class DocumentMeanReciprocalRank:
|
||||
self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]]
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Run the DocumentMeanReciprocalRank on the given inputs.
|
||||
Run the DocumentMRREvaluator on the given inputs.
|
||||
|
||||
`ground_truth_documents` and `retrieved_documents` must have the same length.
|
||||
|
||||
|
@ -1,4 +1,4 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Add DocumentMeanAveragePrecision, it can be used to calculate mean average precision of retrieved documents.
|
||||
Add DocumentMAPEvaluator, it can be used to calculate mean average precision of retrieved documents.
|
||||
|
@ -1,4 +1,4 @@
|
||||
---
|
||||
features:
|
||||
- |
|
||||
Add DocumentMeanReciprocalRank, it can be used to calculate mean reciprocal rank of retrieved documents.
|
||||
Add DocumentMRREvaluator, it can be used to calculate mean reciprocal rank of retrieved documents.
|
||||
|
@ -1,11 +1,11 @@
|
||||
import pytest
|
||||
|
||||
from haystack import Document
|
||||
from haystack.components.evaluators.document_map import DocumentMeanAveragePrecision
|
||||
from haystack.components.evaluators.document_map import DocumentMAPEvaluator
|
||||
|
||||
|
||||
def test_run_with_all_matching():
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
@ -15,7 +15,7 @@ def test_run_with_all_matching():
|
||||
|
||||
|
||||
def test_run_with_no_matching():
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
|
||||
@ -25,7 +25,7 @@ def test_run_with_no_matching():
|
||||
|
||||
|
||||
def test_run_with_partial_matching():
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
|
||||
@ -35,7 +35,7 @@ def test_run_with_partial_matching():
|
||||
|
||||
|
||||
def test_run_with_complex_data():
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[
|
||||
[Document(content="France")],
|
||||
@ -64,14 +64,14 @@ def test_run_with_complex_data():
|
||||
|
||||
def test_run_with_different_lengths():
|
||||
with pytest.raises(ValueError):
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")]],
|
||||
retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
evaluator = DocumentMeanAveragePrecision()
|
||||
evaluator = DocumentMAPEvaluator()
|
||||
evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Berlin")]],
|
||||
|
@ -1,11 +1,11 @@
|
||||
import pytest
|
||||
|
||||
from haystack import Document
|
||||
from haystack.components.evaluators.document_mrr import DocumentMeanReciprocalRank
|
||||
from haystack.components.evaluators.document_mrr import DocumentMRREvaluator
|
||||
|
||||
|
||||
def test_run_with_all_matching():
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
@ -15,7 +15,7 @@ def test_run_with_all_matching():
|
||||
|
||||
|
||||
def test_run_with_no_matching():
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
|
||||
@ -25,7 +25,7 @@ def test_run_with_no_matching():
|
||||
|
||||
|
||||
def test_run_with_partial_matching():
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
|
||||
@ -35,7 +35,7 @@ def test_run_with_partial_matching():
|
||||
|
||||
|
||||
def test_run_with_complex_data():
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
result = evaluator.run(
|
||||
ground_truth_documents=[
|
||||
[Document(content="France")],
|
||||
@ -68,14 +68,14 @@ def test_run_with_complex_data():
|
||||
|
||||
def test_run_with_different_lengths():
|
||||
with pytest.raises(ValueError):
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")]],
|
||||
retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
evaluator = DocumentMeanReciprocalRank()
|
||||
evaluator = DocumentMRREvaluator()
|
||||
evaluator.run(
|
||||
ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
|
||||
retrieved_documents=[[Document(content="Berlin")]],
|
||||
|
Loading…
x
Reference in New Issue
Block a user