From 8b8a93bc0d088b407d0ba1b6a04f61d32040c8e3 Mon Sep 17 00:00:00 2001 From: Silvano Cerza <3314350+silvanocerza@users.noreply.github.com> Date: Thu, 4 Apr 2024 17:04:59 +0200 Subject: [PATCH] refactor: Rename `DocumentMeanAveragePrecision` and `DocumentMeanReciprocalRank` (#7470) * Rename DocumentMeanAveragePrecision and DocumentMeanReciprocalRank * Update releasenotes * Simplify names --- haystack/components/evaluators/document_map.py | 8 ++++---- haystack/components/evaluators/document_mrr.py | 8 ++++---- .../document-map-evaluator-de896c94b54fe3fa.yaml | 2 +- .../document-mrr-evaluator-fa7c266cc91201a7.yaml | 2 +- test/components/evaluators/test_document_map.py | 14 +++++++------- test/components/evaluators/test_document_mrr.py | 14 +++++++------- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/haystack/components/evaluators/document_map.py b/haystack/components/evaluators/document_map.py index 483a7699c..6d30ebbdb 100644 --- a/haystack/components/evaluators/document_map.py +++ b/haystack/components/evaluators/document_map.py @@ -4,20 +4,20 @@ from haystack import Document, component @component -class DocumentMeanAveragePrecision: +class DocumentMAPEvaluator: """ Evaluator that calculates the mean average precision of the retrieved documents, a metric that measures how high retrieved documents are ranked. Each question can have multiple ground truth documents and multiple retrieved documents. - `DocumentMeanAveragePrecision` doesn't normalize its inputs, the `DocumentCleaner` component + `DocumentMAPEvaluator` doesn't normalize its inputs, the `DocumentCleaner` component should be used to clean and normalize the documents before passing them to this evaluator. Usage example: ```python from haystack.components.evaluators import AnswerExactMatchEvaluator - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() result = evaluator.run( ground_truth_documents=[ [Document(content="France")], @@ -41,7 +41,7 @@ class DocumentMeanAveragePrecision: self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]] ) -> Dict[str, Any]: """ - Run the DocumentMeanAveragePrecision on the given inputs. + Run the DocumentMAPEvaluator on the given inputs. All lists must have the same length. :param ground_truth_documents: diff --git a/haystack/components/evaluators/document_mrr.py b/haystack/components/evaluators/document_mrr.py index d0194902a..ab8bcc138 100644 --- a/haystack/components/evaluators/document_mrr.py +++ b/haystack/components/evaluators/document_mrr.py @@ -4,20 +4,20 @@ from haystack import Document, component @component -class DocumentMeanReciprocalRank: +class DocumentMRREvaluator: """ Evaluator that calculates the mean reciprocal rank of the retrieved documents. MRR measures how high the first retrieved document is ranked. Each question can have multiple ground truth documents and multiple retrieved documents. - `DocumentMeanReciprocalRank` doesn't normalize its inputs, the `DocumentCleaner` component + `DocumentMRREvaluator` doesn't normalize its inputs, the `DocumentCleaner` component should be used to clean and normalize the documents before passing them to this evaluator. Usage example: ```python from haystack.components.evaluators import AnswerExactMatchEvaluator - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() result = evaluator.run( ground_truth_documents=[ [Document(content="France")], @@ -40,7 +40,7 @@ class DocumentMeanReciprocalRank: self, ground_truth_documents: List[List[Document]], retrieved_documents: List[List[Document]] ) -> Dict[str, Any]: """ - Run the DocumentMeanReciprocalRank on the given inputs. + Run the DocumentMRREvaluator on the given inputs. `ground_truth_documents` and `retrieved_documents` must have the same length. diff --git a/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml b/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml index 36e28b73d..c7b853237 100644 --- a/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml +++ b/releasenotes/notes/document-map-evaluator-de896c94b54fe3fa.yaml @@ -1,4 +1,4 @@ --- features: - | - Add DocumentMeanAveragePrecision, it can be used to calculate mean average precision of retrieved documents. + Add DocumentMAPEvaluator, it can be used to calculate mean average precision of retrieved documents. diff --git a/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml b/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml index 7e56e9489..152fea593 100644 --- a/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml +++ b/releasenotes/notes/document-mrr-evaluator-fa7c266cc91201a7.yaml @@ -1,4 +1,4 @@ --- features: - | - Add DocumentMeanReciprocalRank, it can be used to calculate mean reciprocal rank of retrieved documents. + Add DocumentMRREvaluator, it can be used to calculate mean reciprocal rank of retrieved documents. diff --git a/test/components/evaluators/test_document_map.py b/test/components/evaluators/test_document_map.py index f203dd01a..9bdfa5631 100644 --- a/test/components/evaluators/test_document_map.py +++ b/test/components/evaluators/test_document_map.py @@ -1,11 +1,11 @@ import pytest from haystack import Document -from haystack.components.evaluators.document_map import DocumentMeanAveragePrecision +from haystack.components.evaluators.document_map import DocumentMAPEvaluator def test_run_with_all_matching(): - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() result = evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]], @@ -15,7 +15,7 @@ def test_run_with_all_matching(): def test_run_with_no_matching(): - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() result = evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Paris")], [Document(content="London")]], @@ -25,7 +25,7 @@ def test_run_with_no_matching(): def test_run_with_partial_matching(): - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() result = evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]], @@ -35,7 +35,7 @@ def test_run_with_partial_matching(): def test_run_with_complex_data(): - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() result = evaluator.run( ground_truth_documents=[ [Document(content="France")], @@ -64,14 +64,14 @@ def test_run_with_complex_data(): def test_run_with_different_lengths(): with pytest.raises(ValueError): - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() evaluator.run( ground_truth_documents=[[Document(content="Berlin")]], retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]], ) with pytest.raises(ValueError): - evaluator = DocumentMeanAveragePrecision() + evaluator = DocumentMAPEvaluator() evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")]], diff --git a/test/components/evaluators/test_document_mrr.py b/test/components/evaluators/test_document_mrr.py index 959492c64..a891ba3f4 100644 --- a/test/components/evaluators/test_document_mrr.py +++ b/test/components/evaluators/test_document_mrr.py @@ -1,11 +1,11 @@ import pytest from haystack import Document -from haystack.components.evaluators.document_mrr import DocumentMeanReciprocalRank +from haystack.components.evaluators.document_mrr import DocumentMRREvaluator def test_run_with_all_matching(): - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() result = evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]], @@ -15,7 +15,7 @@ def test_run_with_all_matching(): def test_run_with_no_matching(): - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() result = evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Paris")], [Document(content="London")]], @@ -25,7 +25,7 @@ def test_run_with_no_matching(): def test_run_with_partial_matching(): - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() result = evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]], @@ -35,7 +35,7 @@ def test_run_with_partial_matching(): def test_run_with_complex_data(): - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() result = evaluator.run( ground_truth_documents=[ [Document(content="France")], @@ -68,14 +68,14 @@ def test_run_with_complex_data(): def test_run_with_different_lengths(): with pytest.raises(ValueError): - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() evaluator.run( ground_truth_documents=[[Document(content="Berlin")]], retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]], ) with pytest.raises(ValueError): - evaluator = DocumentMeanReciprocalRank() + evaluator = DocumentMRREvaluator() evaluator.run( ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]], retrieved_documents=[[Document(content="Berlin")]],