| 335 | return result |
| 336 | |
| 337 | def split_on_tokens(tok_list, text): |
| 338 | if not text.strip(): |
| 339 | return [] |
| 340 | if not tok_list: |
| 341 | return self._tokenize(text) |
| 342 | |
| 343 | tokenized_text = [] |
| 344 | text_list = [text] |
| 345 | for tok in tok_list: |
| 346 | tokenized_text = [] |
| 347 | for sub_text in text_list: |
| 348 | if sub_text not in self.unique_no_split_tokens: |
| 349 | tokenized_text += split_on_token(tok, sub_text) |
| 350 | else: |
| 351 | tokenized_text += [sub_text] |
| 352 | text_list = tokenized_text |
| 353 | |
| 354 | return list( |
| 355 | itertools.chain.from_iterable( |
| 356 | ( |
| 357 | self._tokenize(token) if token not in self.unique_no_split_tokens else [token] |
| 358 | for token in tokenized_text |
| 359 | ) |
| 360 | ) |
| 361 | ) |
| 362 | |
| 363 | no_split_token = self.unique_no_split_tokens |
| 364 | tokenized_text = split_on_tokens(no_split_token, text) |