| 176 | self.sp_model.Load(sp_model_dir) |
| 177 | |
| 178 | def cut(self, chars): # pylint: disable=doc-string-missing |
| 179 | words = [] |
| 180 | idx = 0 |
| 181 | while idx < len(chars): |
| 182 | matched = False |
| 183 | for i in range(self.window_size, 0, -1): |
| 184 | cand = chars[idx:idx + i] |
| 185 | if cand in self.dict: |
| 186 | words.append(cand) |
| 187 | matched = True |
| 188 | break |
| 189 | if not matched: |
| 190 | i = 1 |
| 191 | words.append(chars[idx]) |
| 192 | idx += i |
| 193 | return words |
| 194 | |
| 195 | def tokenize(self, text, unk_token="[UNK]"): # pylint: disable=doc-string-missing |
| 196 | text = convert_to_unicode(text) |