diff --git a/haystack/components/evaluators/sas_evaluator.py b/haystack/components/evaluators/sas_evaluator.py index 7625b837c..0fc5e80da 100644 --- a/haystack/components/evaluators/sas_evaluator.py +++ b/haystack/components/evaluators/sas_evaluator.py @@ -157,6 +157,9 @@ class SASEvaluator: if len(ground_truth_answers) != len(predicted_answers): raise ValueError("The number of predictions and labels must be the same.") + if any(answer is None for answer in predicted_answers): + raise ValueError("Predicted answers must not contain None values.") + if len(predicted_answers) == 0: return {"score": 0.0, "individual_scores": [0.0]} diff --git a/releasenotes/notes/check-for-None-SAS-eval-0b982ccc1491ee83.yaml b/releasenotes/notes/check-for-None-SAS-eval-0b982ccc1491ee83.yaml new file mode 100644 index 000000000..d12adfde0 --- /dev/null +++ b/releasenotes/notes/check-for-None-SAS-eval-0b982ccc1491ee83.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + `SASEvaluator` now raises a `ValueError` if a `None` value is contained in the `predicted_answers` input. diff --git a/test/components/evaluators/test_sas_evaluator.py b/test/components/evaluators/test_sas_evaluator.py index 4a9ee402f..d0e2e9b8b 100644 --- a/test/components/evaluators/test_sas_evaluator.py +++ b/test/components/evaluators/test_sas_evaluator.py @@ -73,6 +73,21 @@ class TestSASEvaluator: with pytest.raises(ValueError): evaluator.run(ground_truth_answers=ground_truths, predicted_answers=predictions) + def test_run_with_none_in_predictions(self): + evaluator = SASEvaluator() + ground_truths = [ + "A construction budget of US $2.3 billion", + "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.", + "The Meiji Restoration in 1868 transformed Japan into a modernized world power.", + ] + predictions = [ + "A construction budget of US $2.3 billion", + None, + "The Meiji Restoration in 1868 transformed Japan into a modernized world power.", + ] + with pytest.raises(ValueError): + evaluator.run(ground_truth_answers=ground_truths, predicted_answers=predictions) + def test_run_not_warmed_up(self): evaluator = SASEvaluator() ground_truths = [