(self, text)
| 76 | return text_list, length |
| 77 | |
| 78 | def bpe_encode(self, text): |
| 79 | if len(text) == 0: |
| 80 | return None |
| 81 | token = self.bpe_tokenizer(text)['input_ids'] |
| 82 | text_list = [1] + token + [2] |
| 83 | if len(text_list) == 0 or len(text_list) > self.batch_max_length: |
| 84 | return None |
| 85 | text_list = text_list + [self.dict[self.GO] |
| 86 | ] * (self.batch_max_length - len(text_list)) |
| 87 | return text_list |
| 88 | |
| 89 | def wp_encode(self, text): |
| 90 | wp_target = self.wp_tokenizer([text], |