Tokenizes paragraphs.
(
sentence_tokenizer: nltk.tokenize.api.TokenizerI,
processor: spm.SentencePieceProcessor,
text: Text,
paragraph_metric: Optional[metrics.Metrics.DelegatingDistribution] = None,
sentence_metric: Optional[metrics.Metrics.DelegatingDistribution] = None
)
| 101 | |
| 102 | |
| 103 | def make_paragraph( |
| 104 | sentence_tokenizer: nltk.tokenize.api.TokenizerI, |
| 105 | processor: spm.SentencePieceProcessor, |
| 106 | text: Text, |
| 107 | paragraph_metric: Optional[metrics.Metrics.DelegatingDistribution] = None, |
| 108 | sentence_metric: Optional[metrics.Metrics.DelegatingDistribution] = None |
| 109 | ) -> Paragraph: |
| 110 | """Tokenizes paragraphs.""" |
| 111 | paragraph_size = 0 |
| 112 | sentences = [] |
| 113 | for sentence in sentence_tokenizer.tokenize(text): |
| 114 | sentencepiece_text = sentencepiece_pb2.SentencePieceText.FromString( |
| 115 | processor.EncodeAsSerializedProto(sentence)) |
| 116 | paragraph_size += len(sentencepiece_text.pieces) |
| 117 | sentences.append(sentencepiece_text) |
| 118 | if sentence_metric: |
| 119 | sentence_metric.update(len(sentencepiece_text.pieces)) |
| 120 | if paragraph_metric: |
| 121 | paragraph_metric.update(paragraph_size) |
| 122 | return Paragraph(sentences=sentences, size=paragraph_size) |
| 123 | |
| 124 | |
| 125 | def read_question_answers(json_path: Text) -> List[QuestionAnswer]: |
no test coverage detected