Align answer span to text with given tokens.
(features: Features, answer_set: Optional[Set[Text]],
processor: spm.SentencePieceProcessor,
span: AnswerSpan)
| 185 | |
| 186 | |
| 187 | def realign_answer_span(features: Features, answer_set: Optional[Set[Text]], |
| 188 | processor: spm.SentencePieceProcessor, |
| 189 | span: AnswerSpan) -> Optional[AnswerSpan]: |
| 190 | """Align answer span to text with given tokens.""" |
| 191 | i = bisect.bisect_left(features.token_offsets, span.begin) |
| 192 | if i == len(features.token_offsets) or span.begin < features.token_offsets[i]: |
| 193 | i -= 1 |
| 194 | j = i + 1 |
| 195 | answer_end = span.begin + len(span.text.encode('utf-8')) |
| 196 | while (j < len(features.token_offsets) and |
| 197 | features.token_offsets[j] < answer_end): |
| 198 | j += 1 |
| 199 | j -= 1 |
| 200 | sp_answer = ( |
| 201 | features.context[features.token_offsets[i]:features.token_offsets[j + 1]] |
| 202 | if j + 1 < len(features.token_offsets) else |
| 203 | features.context[features.token_offsets[i]:]) |
| 204 | if (processor.IdToPiece(features.token_ids[i]).startswith('▁') and |
| 205 | features.token_offsets[i] > 0): |
| 206 | sp_answer = sp_answer[1:] |
| 207 | sp_answer = evaluation.normalize_answer(sp_answer.decode('utf-8')) |
| 208 | if answer_set is not None and sp_answer not in answer_set: |
| 209 | # No need to warn if the cause was breaking word boundaries. |
| 210 | if len(sp_answer) and not len(sp_answer) > len( |
| 211 | evaluation.normalize_answer(span.text)): |
| 212 | logging.warning('%s: "%s" not in %s.', features.question_id, sp_answer, |
| 213 | answer_set) |
| 214 | return None |
| 215 | return AnswerSpan(begin=i, end=j, text=span.text) |
| 216 | |
| 217 | |
| 218 | def read_sentencepiece_model(path): |
no test coverage detected