yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-11-12 16:14:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: Use add_isolated_node_eval of eval_batch in run_batch (#5223)

Browse Source 
* Fix isolated node eval in eval_batch

* Add unit test
This commit is contained in:
bogdankostic 2023-06-28 16:51:23 +02:00 committed by GitHub
parent bc86f57715
commit ed1bad1155
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
haystack/pipelines/base.py
View File

@ -1366,6 +1366,9 @@ class Pipeline:
event_name="Evaluation", event_name="Evaluation",
event_properties={"pipeline.classname": self.__class__.__name__, "pipeline.config_hash": self.config_hash}, event_properties={"pipeline.classname": self.__class__.__name__, "pipeline.config_hash": self.config_hash},
) )
if add_isolated_node_eval:
params = {} if params is None else params.copy()
params["add_isolated_node_eval"] = True
predictions_batches = self.run_batch( predictions_batches = self.run_batch(
queries=[label.query for label in labels], labels=labels, documents=documents, params=params, debug=True queries=[label.query for label in labels], labels=labels, documents=documents, params=params, debug=True
@ -1373,11 +1376,9 @@ class Pipeline:
eval_result = self._generate_eval_result_from_batch_preds( eval_result = self._generate_eval_result_from_batch_preds(
predictions_batches=predictions_batches, predictions_batches=predictions_batches,
params=params,
sas_model_name_or_path=sas_model_name_or_path, sas_model_name_or_path=sas_model_name_or_path,
sas_batch_size=sas_batch_size, sas_batch_size=sas_batch_size,
sas_use_gpu=sas_use_gpu, sas_use_gpu=sas_use_gpu,
add_isolated_node_eval=add_isolated_node_eval,
custom_document_id_field=custom_document_id_field, custom_document_id_field=custom_document_id_field,
context_matching_min_length=context_matching_min_length, context_matching_min_length=context_matching_min_length,
context_matching_boost_split_overlaps=context_matching_boost_split_overlaps, context_matching_boost_split_overlaps=context_matching_boost_split_overlaps,
@ -1390,11 +1391,9 @@ class Pipeline:
def _generate_eval_result_from_batch_preds( def _generate_eval_result_from_batch_preds(
self, self,
predictions_batches: Dict, predictions_batches: Dict,
params: Optional[dict] = None,
sas_model_name_or_path: Optional[str] = None, sas_model_name_or_path: Optional[str] = None,
sas_batch_size: int = 32, sas_batch_size: int = 32,
sas_use_gpu: bool = True, sas_use_gpu: bool = True,
add_isolated_node_eval: bool = False,
custom_document_id_field: Optional[str] = None, custom_document_id_field: Optional[str] = None,
context_matching_min_length: int = 100, context_matching_min_length: int = 100,
context_matching_boost_split_overlaps: bool = True, context_matching_boost_split_overlaps: bool = True,
@ -1402,9 +1401,6 @@ class Pipeline:
use_auth_token: Optional[Union[str, bool]] = None, use_auth_token: Optional[Union[str, bool]] = None,
) -> EvaluationResult: ) -> EvaluationResult:
eval_result = EvaluationResult() eval_result = EvaluationResult()
if add_isolated_node_eval:
params = {} if params is None else params.copy()
params["add_isolated_node_eval"] = True
for node_name in predictions_batches["_debug"].keys(): for node_name in predictions_batches["_debug"].keys():
node_output = predictions_batches["_debug"][node_name]["output"] node_output = predictions_batches["_debug"][node_name]["output"]

12
test/pipelines/test_eval_batch.py
View File

@ -1,4 +1,6 @@
import logging import logging
from unittest.mock import patch
import pytest import pytest
import sys import sys
from copy import deepcopy from copy import deepcopy
@ -21,6 +23,16 @@ from haystack.nodes.translator.transformers import TransformersTranslator
from haystack.schema import Answer, Document, EvaluationResult, Label, MultiLabel, Span from haystack.schema import Answer, Document, EvaluationResult, Label, MultiLabel, Span
@pytest.mark.unit
@patch("haystack.pipelines.base.Pipeline.run_batch")
def test_eval_batch_add_isolated_node_eval_passed_to_run_batch(mock_run_batch):
pipeline = Pipeline()
pipeline.eval_batch(labels=EVAL_LABELS, add_isolated_node_eval=True)
_, kwargs = mock_run_batch.call_args
assert "add_isolated_node_eval" in kwargs["params"]
assert kwargs["params"]["add_isolated_node_eval"] is True
@pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner") @pytest.mark.skipif(sys.platform in ["win32", "cygwin"], reason="Causes OOM on windows github runner")
@pytest.mark.parametrize("document_store_with_docs", ["memory"], indirect=True) @pytest.mark.parametrize("document_store_with_docs", ["memory"], indirect=True)
@pytest.mark.parametrize("retriever_with_docs", ["embedding"], indirect=True) @pytest.mark.parametrize("retriever_with_docs", ["embedding"], indirect=True)