convert text-label into text-index.
(self, text)
| 56 | return dict_character |
| 57 | |
| 58 | def encode(self, text): |
| 59 | """ convert text-label into text-index. |
| 60 | """ |
| 61 | if len(text) == 0: |
| 62 | return None, None |
| 63 | if self.lower: |
| 64 | text = text.lower() |
| 65 | length = len(text) |
| 66 | text = [self.GO] + list(text) + [self.SPACE] |
| 67 | text_list = [] |
| 68 | for char in text: |
| 69 | if char not in self.dict: |
| 70 | continue |
| 71 | text_list.append(self.dict[char]) |
| 72 | if len(text_list) == 0 or len(text_list) > self.batch_max_length: |
| 73 | return None, None |
| 74 | text_list = text_list + [self.dict[self.GO] |
| 75 | ] * (self.batch_max_length - len(text_list)) |
| 76 | return text_list, length |
| 77 | |
| 78 | def bpe_encode(self, text): |
| 79 | if len(text) == 0: |