Generate Text Span :param generated: :param prefix_tree: :param src_sentence: :param end_sequence_search_tokens: :return:
(self, generated: List[str], prefix_tree: Dict,
end_sequence_search_tokens: List[str] = None)
| 290 | return state, last_special_index |
| 291 | |
| 292 | def search_prefix_tree(self, generated: List[str], prefix_tree: Dict, |
| 293 | end_sequence_search_tokens: List[str] = None): |
| 294 | """ |
| 295 | Generate Text Span |
| 296 | :param generated: |
| 297 | :param prefix_tree: |
| 298 | :param src_sentence: |
| 299 | :param end_sequence_search_tokens: |
| 300 | :return: |
| 301 | """ |
| 302 | tree = prefix_tree |
| 303 | for index, token in enumerate(generated): |
| 304 | tree = tree[token] |
| 305 | is_tree_end = len(tree) == 1 and self.tree_end in tree |
| 306 | |
| 307 | if is_tree_end: |
| 308 | return end_sequence_search_tokens |
| 309 | |
| 310 | if self.tree_end in tree: |
| 311 | return end_sequence_search_tokens |
| 312 | |
| 313 | valid_token = list(tree.keys()) |
| 314 | return valid_token |
| 315 | |
| 316 | def get_state_valid_tokens(self, src_sentence, tgt_generated): |
| 317 | """ |
no outgoing calls
no test coverage detected