(self, text)
| 50 | return dict_character |
| 51 | |
| 52 | def bpe_encode(self, text): |
| 53 | if len(text) == 0: |
| 54 | return None, None |
| 55 | token = self.bpe_tokenizer(text)['input_ids'] |
| 56 | length = len(token) |
| 57 | text_list = [self.bos_idx] + token + [2] |
| 58 | if len(text_list) == 0 or len(text_list) > self.batch_max_length: |
| 59 | return None, None |
| 60 | text_list = text_list + [self.padding_idx |
| 61 | ] * (self.batch_max_length - len(text_list)) |
| 62 | return text_list, length |