| 272 | self._sentencepiece_model_path) |
| 273 | |
| 274 | def _make_features(self, stride_index: int, paragraph_texts: List[Text], |
| 275 | paragraphs: List[Paragraph], |
| 276 | question_answer_evidence: QuestionAnswerEvidence, |
| 277 | ids: List[int], |
| 278 | paragraph_offset: int) -> Tuple[int, Features]: |
| 279 | global_ids = ( |
| 280 | [self._sentencepiece_processor.PieceToId(_CLS_PIECE)] + |
| 281 | [self._sentencepiece_processor.PieceToId(_QUESTION_PIECE)] * len(ids)) |
| 282 | segment_ids = [i + 1 for i in range(len(ids))] # offset for CLS token |
| 283 | token_ids, sentences = [], [] |
| 284 | offsets, offset, full_text = [-1] * len(ids), 0, True |
| 285 | for i in range(paragraph_offset, len(paragraph_texts)): |
| 286 | if i < len(paragraphs): |
| 287 | paragraph = paragraphs[i] |
| 288 | else: |
| 289 | paragraphs.append( |
| 290 | make_paragraph( |
| 291 | self._sentence_tokenizer, |
| 292 | self._sentencepiece_processor, |
| 293 | paragraph_texts[i], |
| 294 | paragraph_metric=metrics.Metrics.distribution( |
| 295 | '_', 'paragraphs'), |
| 296 | sentence_metric=metrics.Metrics.distribution('_', 'sentences'))) |
| 297 | paragraph = paragraphs[-1] |
| 298 | for sentence in paragraph.sentences: |
| 299 | if (len(ids) + len(token_ids) + len(sentence.pieces) + 1 >= |
| 300 | self._max_num_tokens or |
| 301 | len(global_ids) >= self._max_num_global_tokens): |
| 302 | full_text = False |
| 303 | break |
| 304 | for j, piece in enumerate(sentence.pieces): |
| 305 | token_ids.append(piece.id) |
| 306 | segment_ids.append(len(global_ids)) |
| 307 | offsets.append(offset + piece.begin) |
| 308 | if j == 0 and sentences: |
| 309 | offsets[-1] -= 1 |
| 310 | offset += len(sentence.text.encode('utf-8')) + 1 |
| 311 | global_ids.append(self._sentencepiece_processor.PieceToId(_EOS_PIECE)) |
| 312 | sentences.append(sentence.text) |
| 313 | if not full_text: |
| 314 | break |
| 315 | context = ' '.join(sentences).encode('utf-8') |
| 316 | token_ids.append(self._sentencepiece_processor.PieceToId(_NULL_PIECE)) |
| 317 | offsets.append(len(context)) |
| 318 | segment_ids.append(0) |
| 319 | next_paragraph_index = len(paragraph_texts) |
| 320 | if not full_text and self._stride > 0: |
| 321 | shift = paragraphs[paragraph_offset].size |
| 322 | next_paragraph_index = paragraph_offset + 1 |
| 323 | while (next_paragraph_index < len(paragraphs) and |
| 324 | shift + paragraphs[next_paragraph_index].size <= self._stride): |
| 325 | shift += paragraphs[next_paragraph_index].size |
| 326 | next_paragraph_index += 1 |
| 327 | return next_paragraph_index, Features( |
| 328 | id='{}--{}'.format(question_answer_evidence.question.id, |
| 329 | question_answer_evidence.evidence.info.id), |
| 330 | stride_index=stride_index, |
| 331 | question_id=question_answer_evidence.question.id, |