yujunjun/haystack
1
0
Fork 0
mirror of https://github.com/deepset-ai/haystack.git synced 2025-09-07 07:13:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #26 from deepset-ai/no_answer

Browse Source 
Add no answer handling
This commit is contained in:
Timo Moeller 2020-02-24 16:27:59 +01:00 committed by GitHub
commit c0910c82c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 72 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
haystack/__init__.py
View File

@ -46,7 +46,7 @@ class Finder:
# 3) Apply reader to get granular answer(s)
logger.info(f"Applying the reader now to look for the answer in detail ...")
results = self.reader.predict(question=question,
paragrahps=paragraphs,
paragraphs=paragraphs,
meta_data_paragraphs=meta_data,
top_k=top_k_reader)

116
haystack/reader/farm.py
View File

@ -27,10 +27,10 @@ class FARMReader:
self,
model_name_or_path,
context_window_size=30,
no_ans_boost=-100,
batch_size=16,
batch_size=50,
use_gpu=True,
n_candidates_per_passage=2):
no_ans_boost=None,
n_candidates_per_paragraph=1):
"""
:param model_name_or_path: directory of a saved model or the name of a public model:
- 'bert-base-cased'
@ -40,25 +40,31 @@ class FARMReader:
....
See https://huggingface.co/models for full list of available models.
:param context_window_size: The size, in characters, of the window around the answer span that is used when displaying the context around the answer.
:param no_ans_boost: How much the no_answer logit is boosted/increased.
The higher the value, the more likely a "no answer possible" is returned by the model
:param batch_size: Number of samples the model receives in one batch for inference
Memory consumption is much lower in inference mode. Recommendation: increase the batch size to a value so only a single batch is used.
:param use_gpu: Whether to use GPU (if available)
:param n_candidates_per_passage: How many candidate answers are extracted per text sequence that the model can process at once (depends on `max_seq_len`).
Note: This is not the number of "final answers" you will receive
(see `top_k` in FARMReader.predict() or Finder.get_answers() for that)
# TODO adjust farm. n_cand = 2 returns no answer + highest positive answer
# should return no answer + 2 best positive answers
# drawback: answers from a single paragraph might be very similar in text and score
# we need to have more varied answers (by excluding overlapping answers?)
:param no_ans_boost: How much the no_answer logit is boosted/increased.
Possible values: None (default) = disable returning "no answer" predictions
Negative = lower chance of "no answer" being predicted
Positive = increase chance of "no answer"
:param n_candidates_per_paragraph: How many candidate answers are extracted per text sequence that the model can process at once (depends on `max_seq_len`).
Note: - This is not the number of "final answers" you will receive
(see `top_k` in FARMReader.predict() or Finder.get_answers() for that)
- FARM includes no_answer in the sorted list of predictions
"""
if no_ans_boost is None:
no_ans_boost = 0
self.return_no_answers = False
else:
self.return_no_answers = True
self.n_candidates_per_paragraph = n_candidates_per_paragraph
self.inferencer = Inferencer.load(model_name_or_path, batch_size=batch_size, gpu=use_gpu, task_type="question_answering")
self.inferencer.model.prediction_heads[0].context_window_size = context_window_size
self.inferencer.model.prediction_heads[0].no_ans_threshold = no_ans_boost # TODO adjust naming and concept in FARM
self.no_ans_boost = no_ans_boost
self.inferencer.model.prediction_heads[0].n_best = n_candidates_per_passage
self.inferencer.model.prediction_heads[0].no_ans_boost = no_ans_boost
self.inferencer.model.prediction_heads[0].n_best = n_candidates_per_paragraph + 1 # including possible no_answer
def train(self, data_dir, train_filename, dev_filename=None, test_file_name=None,
use_gpu=True, batch_size=10, n_epochs=2, learning_rate=1e-5,
@ -146,7 +152,7 @@ class FARMReader:
self.inferencer.model.save(directory)
self.inferencer.processor.save(directory)
def predict(self, question, paragrahps, meta_data_paragraphs=None, top_k=None, max_processes=1):
def predict(self, question, paragraphs, meta_data_paragraphs=None, top_k=None, max_processes=1):
"""
Use loaded QA model to find answers for a question in the supplied paragraphs.
@ -160,7 +166,7 @@ class FARMReader:
'offset_answer_end': 154,
'probability': 0.9787139466668613,
'score': None,
'document_id': None
'document_id': '1337'
},
...
]
@ -176,19 +182,19 @@ class FARMReader:
"""
if meta_data_paragraphs is None:
meta_data_paragraphs = len(paragrahps) * [None]
assert len(paragrahps) == len(meta_data_paragraphs)
meta_data_paragraphs = len(paragraphs) * [None]
assert len(paragraphs) == len(meta_data_paragraphs)
# convert input to FARM format
input_dicts = []
for paragraph, meta_data in zip(paragrahps, meta_data_paragraphs):
for paragraph, meta_data in zip(paragraphs, meta_data_paragraphs):
cur = {"text": paragraph,
"questions": [question],
"document_id": meta_data["document_id"]
}
input_dicts.append(cur)
# get answers from QA model (Default: top 5 per input paragraph)
# get answers from QA model
predictions = self.inferencer.inference_from_dicts(
dicts=input_dicts, rest_api_schema=True, max_processes=max_processes
)
@ -200,6 +206,8 @@ class FARMReader:
no_ans_gaps = []
best_score_answer = 0
for pred in predictions:
answers_per_paragraph = []
no_ans_gaps.append(pred["predictions"][0]["no_ans_gap"])
for a in pred["predictions"][0]["answers"]:
# skip "no answers" here
if a["answer"]:
@ -210,38 +218,17 @@ class FARMReader:
"offset_start": a["offset_answer_start"] - a["offset_context_start"],
"offset_end": a["offset_answer_end"] - a["offset_context_start"],
"document_id": a["document_id"]}
answers.append(cur)
no_ans_gaps.append(pred["predictions"][0]["no_ans_gap"])
answers_per_paragraph.append(cur)
if a["score"] > best_score_answer:
best_score_answer = a["score"]
# only take n best candidates. Answers coming back from FARM are sorted with decreasing relevance.
answers += answers_per_paragraph[:self.n_candidates_per_paragraph]
# adjust no_ans_gaps
no_ans_gaps = np.array(no_ans_gaps)
no_ans_gaps_adjusted = no_ans_gaps + self.no_ans_boost
# We want to heuristically rank how likely or unlikely the "no answer" option is.
# case: all documents return no answer, then all no_ans_gaps are positive
if np.sum(no_ans_gaps_adjusted < 0) == 0:
# to rank we add the smallest no_ans_gap (a document where an answer would be nearly as likely as the no anser)
# to the highest answer score we found
no_ans_score = best_score_answer + min(no_ans_gaps_adjusted)
# case: documents where answers are preferred over no answer, the no_ans_gap is negative
else:
# the lowest (highest negative) no_ans_gap would be needed as positive no_ans_boost for the
# model to return "no answer" on all documents
# we subtract this value from the best answer score to rank our "no answer" option
# magically this is the same equation as used for the case above : )
no_ans_score = best_score_answer + min(no_ans_gaps_adjusted)
cur = {"answer": "[computer says no answer is likely]",
"score": no_ans_score,
"probability": float(expit(np.asarray(no_ans_score) / 8)), # just a pseudo prob for now
"context": "",
"offset_start": 0,
"offset_end": 0,
"document_id": None}
answers.append(cur)
# Calculate the score for predicting "no answer", relative to our best positive answer score
no_ans_prediction, max_no_ans_gap = self._calc_no_answer(no_ans_gaps,best_score_answer)
if self.return_no_answers:
answers.append(no_ans_prediction)
# sort answers by their `probability` and select top-k
answers = sorted(
@ -249,7 +236,32 @@ class FARMReader:
)
answers = answers[:top_k]
result = {"question": question,
"adjust_no_ans_boost": -min(no_ans_gaps_adjusted),
"no_ans_gap": max_no_ans_gap,
"answers": answers}
return result
@staticmethod
def _calc_no_answer(no_ans_gaps,best_score_answer):
# "no answer" scores and positive answers scores are difficult to compare, because
# + a positive answer score is related to one specific document
# - a "no answer" score is related to all input documents
# Thus we compute the "no answer" score relative to the best possible answer and adjust it by
# the most significant difference between scores.
# Most significant difference: a model switching from predicting an answer to "no answer" (or vice versa).
# No_ans_gap coming from FARM mean how much no_ans_boost should change to switch predictions
no_ans_gaps = np.array(no_ans_gaps)
max_no_ans_gap = np.max(no_ans_gaps)
if (np.sum(no_ans_gaps < 0) == len(no_ans_gaps)): # all passages "no answer" as top score
no_ans_score = best_score_answer - max_no_ans_gap # max_no_ans_gap is negative, so it increases best pos score
else: # case: at least one passage predicts an answer (positive no_ans_gap)
no_ans_score = best_score_answer - max_no_ans_gap
no_ans_prediction = {"answer": None,
"score": no_ans_score,
"probability": float(expit(np.asarray(no_ans_score) / 8)), # just a pseudo prob for now
"context": None,
"offset_start": 0,
"offset_end": 0,
"document_id": None}
return no_ans_prediction, max_no_ans_gap

9
tutorials/Tutorial1_Basic_QA_Pipeline.py
View File

@ -35,10 +35,10 @@ retriever = TfidfRetriever(document_store=document_store)
# A reader scans the text chunks in detail and extracts the k best answers
# Reader use more powerful but slower deep learning models
# You can select a local model or any of the QA models published on huggingface's model hub (https://huggingface.co/models)
# You can select a local model or any of the QA models published on huggingface's model hub (https://huggingface.co/models)
# here: a medium sized BERT QA model trained via FARM on Squad 2.0
# You can adjust the model to return "no answer possible" with the no_ans_boost. Higher values mean the model prefers "no answer possible"
reader = FARMReader(model_name_or_path="deepset/bert-base-cased-squad2", use_gpu=False, no_ans_boost=0)
reader = FARMReader(model_name_or_path="deepset/bert-base-cased-squad2", use_gpu=False)
# OR: use alternatively a reader from huggingface's transformers package (https://github.com/huggingface/transformers)
# reader = TransformersReader(model="distilbert-base-uncased-distilled-squad", tokenizer="distilbert-base-uncased", use_gpu=-1)
@ -51,7 +51,10 @@ finder = Finder(reader, retriever)
# The higher top_k_retriever, the better (but also the slower) your answers.
prediction = finder.get_answers(question="Who is the father of Arya Stark?", top_k_retriever=10, top_k_reader=5)
#prediction = finder.get_answers(question="Who is the daughter of Arya Stark?", top_k_reader=5) # impossible question test
# to test impossible questions we need a large QA model, e.g. deepset/bert-large-uncased-whole-word-masking-squad2
# and we need to enable returning "no answer possible" by setting no_ans_boost=X in FARMReader
# prediction = finder.get_answers(question="Who is the first daughter of Arya Stark?", top_k_retriever=10, top_k_reader=5)
#prediction = finder.get_answers(question="Who created the Dothraki vocabulary?", top_k_reader=5)
#prediction = finder.get_answers(question="Who is the sister of Sansa?", top_k_reader=5)